Merge pull request #16 in PAR/yap from DEV-3068-write-up-a-document-on-converting-yap-to-6.5 to master

* commit '7ce9202e5aaa6cf8240d3997c978e2f8dfdec079':
  Note minimum version for migrating off YAP
  Fix formatting typo in documentation
  Add documentation on migrating to managed mesh

.gitignore

@@ -0,0 +1,2 @@
+.vscode
+docs/

Makefile

@@ -14,5 +14,12 @@ install:
 	test -f $(DESTDIR)$(TOPFILE) || echo "partner:" > $(DESTDIR)$(TOPFILE)
 	grep -q 'aggregator|vxr' $(DESTDIR)$(TOPFILE) || cat salt/top.sls >> $(DESTDIR)$(TOPFILE)
 
+docs: README.rst
+	mkdir -p docs/
+	cp VXLAN-backhaul.png docs/
+	rst2html README.rst docs/index.html
 
-.PHONY: all install
+clean:
+	rm -rf docs/
+
+.PHONY: all install clean

README.rst

@@ -15,6 +15,10 @@ If a backhaul is not already set up in a data center, additional "VXR" boxes
 can be added to each data center to provide an overlay backhaul using
 VXLAN-over-IPSEC.
 
+.. contents::
+
+Installation and setup
+----------------------
 
 Initial installation
 ====================
@@ -23,11 +27,28 @@ First, install the software on the bondingadmin server::
 
     make install
 
+
+.. note:: The rest of the yap commands are run on the management server, unless
+   otherwise stated.
+
 Then add a read-only user in the Bondingadmin web interface allow the tool to
 query the API. Add the user details using the ``yap`` tool::
 
     yap auth-set user@example.com mypassword
 
+Upgrading
+==========
+
+From the directory containing the YAP checkout, usually ~/yap, perform the
+following::
+
+    git pull
+    make install
+    yap upgrade [region]
+
+
+region can be left blank if you wish to upgrade all regions at once.
+
 
 Setting up regions
 ==================
@@ -38,12 +59,12 @@ spaces. To add a region::
     yap region-add yvr
 
 
-Setting up spaces
+Adding spaces
-=================
+=============
 
 To add the space with key ``foo``::
 
-    yap space-add add foo
+    yap space-add foo
 
 
 Setting VLAN region associations
@@ -91,6 +112,14 @@ port::
 
 The necessary software will be installed automatically.
 
+If you want to add global OSPF to the VXR in order to transit non-private WAN
+traffic::
+
+    yap vxr-enable-global yvr-xvr01
+
+If it's enabled and you want to disable it::
+
+    yap vxr-disable-global yvr-xvr01
 
 Adding aggregators
 ==================
@@ -103,6 +132,21 @@ setup a vlan trunk interface, then add it::
 This will install some software on the aggregator to maintain the VLANs and
 OSPF peering on the ``eth1`` trunk port.
 
+To add a space-specific VLAN IP, you need the aggregator ID, the space key,
+and the VLAN IP with the subnet mask. If unset, a default address will be used::
+
+    yap agg-set-space-ip 1 foo 10.7.7.7/30
+
+
+Adding custom BIRD configuration
+================================
+
+To inject custom BIRD configuration through yap for a specific space on an
+aggregator, first write the configuration to a file. To apply the configuration,
+specify the aggregator ID, space key, and the filename::
+
+    yap agg-set-space-bird-config 1 foo bird.conf
+
 
 Showing status
 ==============
@@ -118,3 +162,403 @@ example, to show the state of space ``foo`` on the VXR ``yvr-vxr01`` and the
 aggregator with ID 1::
 
     salt -C 'L@yvr-vxr01,node-1' cmd.run "yap status foo"
+
+
+Architectural overview
+----------------------
+
+The following diagram shows an overview of the various nodes involved in a
+typical YAP deployment for a space. This fictional space has a firewall in
+YVR only, but bonds in both YVR and TOR.
+
+The red circles denote details and troubleshooting commands that can be run
+on each respective node.
+
+.. image:: VXLAN-backhaul.png
+   :scale: 30 %
+   :alt: VXLAN backhaul diagram
+
+.. This diagram may be updated at the following link:
+   https://www.lucidchart.com/invitations/accept/27dfc950-e351-4511-b42a-d1f08fe26833
+
+
+Adding spaces
+-------------
+
+Prerequisites
+=============
+
+* All bonds are moved to yap-enabled aggregators.
+* A VLAN is designated for each region that will host bonds. For example, for
+  a space that has bonds on aggregators in two regions, YVR and TOR, you must
+  designate a VLAN for both regions.
+
+Migrating existing private WAN spaces
+=====================================
+
+The following commands are all to be run on the management server.
+
+.. warning:: There will be a brief outage when migrating a space.
+
+1. Add the space::
+
+    yap space-add <key>
+
+   This can be run in advance as it does not make any runtime changes.
+
+2. To calculate the subnet for each region/space, you can run the following
+   command. This only returns the network that will be designated for the VLAN
+   on the aggregators in the region, it does not apply any changes::
+
+    yap subnet-get <key> <region>
+
+   This will return the base subnet for this space-region pair, as well as the
+   specific IPs of the aggregators in that region. The first IP in the subnet
+   is reserved for the firewall::
+
+    Subnet: 100.31.88.0/21
+    Firewall: 100.31.88.1
+    Aggregators:
+        agg03: 100.31.88.5
+
+3. Configure the firewall with the IP shown in step 2 on the VLAN interface and
+   configure OSPF. While the exact settings will be vendor-specific, here are
+   the general details:
+
+   * area 0.0.0.0
+   * subnet <from step 2>
+   * redistribute connected
+   * hello interval 10s
+   * dead interval 40s
+
+4. Add a VLAN association for each region::
+
+	yap vlan-set <key> <region> <vlan_id>
+
+   This will start the VLAN interfaces on each yap-enabled aggregator in the
+   region using the same subnet reflected in step 2.
+
+   .. caution:: This is the start of an outage for the space, as the private
+      WAN router's BGP protocols for the space are brought down to prevent
+      routing loops/conflicts.
+
+5. Confirm OSPF is up in each region by running this command on the
+   aggregators::
+
+    yap status <key>
+
+   If the OSPF protocol is not 'Running', jump to troubleshooting
+   `B: Aggregator`_.
+
+6. Once OSPF is up and the routes have propagated both ways, you can disable
+   the outbound gateway configured in the existing space to finish cleanup.
+
+
+Adding new private WAN spaces
+=============================
+
+Follow the same steps as for migrating an existing space, with these two
+exceptions:
+
+* Enable private WAN on the space through the management server interface.
+* An outbound gateway should not be enabled in the space's private WAN tab,
+  however, you may wish to add a disabled gateway for record-keeping of the
+  firewall's IP.
+
+Troubleshooting
+---------------
+
+A: Bond
+=======
+
+While YAP doesn't directly affect bonds, it can be useful to troubleshoot
+private WAN routes at the bond level, by inspecting their routing table::
+
+    ip route show table bonding-pwan
+
+B: Aggregator
+=============
+
+YAP-enabled aggregators have a ``yap`` command installed that can be used
+to show information about the spaces currently running on the aggregator.
+
+The most useful command is ``yap status <space key>``, which shows the status
+of the bird protocols and the current routing table for that space::
+
+    agg:~# yap status bammya
+
+    spcbammya BGP      krt8251  up     2018-12-06  Established
+    ospf_bammya OSPF     krt8251  up     07:21:22    Running
+
+    default via 100.109.152.1 dev vl-bammya proto bird
+    10.10.1.0/24 via 100.109.152.8 dev vl-bammya proto bird
+    192.168.33.0/24 via 100.109.152.8 dev vl-bammya proto bird
+
+You can also directly check the status of the systemd service for any given
+space::
+
+    agg:~# systemctl status yap-space@bammya.service
+
+    ● yap-space@bammya.service - YAP space bammya
+    Loaded: loaded (/etc/systemd/system/yap-space@.service; disabled; vendor preset: enabled)
+    Active: active (exited) since Fri 2019-07-12 21:56:56 UTC; 1s ago
+    Process: 1210665 ExecStart=/usr/local/bin/yap check-policy-rules %i (code=exited, status=0/SUCCESS)
+    Process: 1210603 ExecStartPre=/usr/local/bin/yap service-start %i (code=exited, status=0/SUCCESS)
+    Main PID: 1210665 (code=exited, status=0/SUCCESS)
+
+    Jul 12 21:56:56 root-agg yap[1210665]: BIRD 2.0.2 ready.
+    Jul 12 21:56:56 root-agg yap[1210665]: spcbammya_pwr1_ipv6: disabled
+
+You can also use a wildcard to see the status of all spaces, or perform other
+operations on the services::
+
+    agg:~# systemctl restart yap-space@*.service
+
+The BGP protocol for the space is controlled by bonding and should be in
+'Established' state. The ``ospf_<key>`` protocol is the one managed by YAP and
+should be in 'Running' state. If the status is 'Alone' instead, it means there
+are no OSPF neighbors.
+
+If you want to, you can show the current OSPF neighbors for a space::
+
+    pwanbirdc - show ospf neighbor ospf_<key>
+
+An aggregator has one VLAN interface per space, which follows the naming
+convention of ``vl-<key>``. You can use this command to show the VLAN id::
+
+    ip -d link show dev vl-bammya
+
+Lastly, you can look at the VLAN interface to see the aggregator's IP, as well
+as the subnet designated for the space and routing group::
+
+    agg:~# ip address show dev vl-bammya
+
+    440: vl-bammya@eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
+        link/ether d0:43:1e:c5:1b:44 brd ff:ff:ff:ff:ff:ff
+        inet 100.109.152.7/21 scope global vl-bammya
+
+In the example above, the firewall would be configured with ``100.109.152.1/21``.
+
+Knowing the subnet, you can test ICMP connectivity to the firewall IP::
+
+    ping <gateway IP>
+
+When troubleshooting OSPF it may be useful to run a packet capture on the VLAN
+interface to see which options are set::
+
+    tcpdump -ni vl-<key> proto 89 -vvv
+
+
+D: VXR
+======
+
+The most useful command is ``yap status <space key>``, which shows the status
+of the bird protocol and the current routing table for that space::
+
+    agg:~# yap status bammya
+
+    ospf_bammya OSPF       bammya up     07:21:23.175  Running
+
+    default via 100.109.152.1 dev vl-bammya proto bird metric 32
+    10.10.1.0/24 via 100.109.152.8 dev vl-bammya proto bird metric 32
+
+Otherwise, the same troubleshooting steps apply as on the aggregator.
+
+If you need to troubleshoot the VXLAN as well, you can view the interface
+details with the standard linux utilities::
+
+    agg:~# ip -d l show dev vx-<key>
+
+    191: vx-bammya: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1432 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
+        link/ether 66:da:5c:17:37:38 brd ff:ff:ff:ff:ff:ff promiscuity 0
+        vxlan id 59 srcport 0 0 dstport 4789 ageing 300 udpcsum noudp6zerocsumtx noudp6zerocsumrx addrgenmode eui64 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+
+E: Firewall
+===========
+
+Out of YAP's control. Here be dragons.
+
+F: bondingadmin
+===============
+
+Like all the nodes, there is a command in the path called ``yap`` that serves
+as the entry point for all things backhauled. Most of the commands are
+described above in their relevant sections. You can always run ``yap`` with
+no arguments to see what actions are available::
+
+	root@bondingadmin:~# yap
+	/usr/local/bin/yap <action> [args]
+
+	Actions:
+
+	region-list
+	region-show <region>
+	region-add <region>
+	...
+
+
+Migrating a YAP space to a managed mesh space
+-----------------------------------------------
+
+As of 6.5, a successor to YAP is properly available in bonding in the form
+of the new private WAN modes (without PWRs) along with aggregator
+interfaces, addresses, and protocols.
+
+Migrating to managed mesh or unmanaged private WAN is required for continued
+support, and can be done with minimal downtime given the appropriate preparation.
+
+.. note::
+
+    To migrate a space to YAP, all aggregators carrying space traffic must be
+    upgraded to bonding version 6.5 or later.
+
+
+Preface
+============
+
+Recall that YAP has the following sets of objects::
+
+    A (aggregators)
+    D (device names)
+    R (regions)
+    S (spaces)
+    VID (VLAN IDs)
+    IP (PWAN IPs)
+
+and that these objects are related by the following functions::
+
+    r: A → R
+    d: A → D
+    v: S x R → VID
+    i: S x A → IP
+
+Given these sets and maps, YAP works by doing the following for each space *s*
+and aggregator *a*:
+
+#. Create a VLAN interface on *d(a)* having VLAN ID *v(s, r(a))*
+#. Add address *i(s, a)* to that VLAN interface.
+#. Run OSPF on that VLAN interface.
+
+Additionally, optional custom BIRD configuration can be defined for a space on a
+particular aggregator, i.e. there is an optional YAP object::
+
+    B (Custom space BIRD configuration)
+
+with relation::
+
+    b: S x A → B
+
+To migrate from YAP to a managed mesh, we need to recreate the same objects,
+i.e. for each space *s* and aggregator *a* we need to:
+
+0. Create trunk interface *d(a)* on aggregator *a*
+   (this only needs to be done once for *a*).
+#. Create a VLAN interface on *d(a)* with VID *v(s, r(a))*.
+#. Add interface IP *i(s, a)* to that VLAN interface.
+#. Create an OSPF protocol configured to have an area with that VLAN interface.
+
+Preparation
+================
+
+The instructions in this section are for preparing to migrate from YAP to
+managed mesh for a single private WAN space, one aggregator at a time.
+
+Let **S** be the YAP space to be migrated,
+let **A** be the aggregator to be migrated,
+and let **R** be the region **A** belongs to.
+
+.. tip::
+
+    All YAP commands given are run on the management server,
+    and all aggregator objects (interfaces, addresses, and protocols)
+    are created through the management server on the aggregator details
+    page.
+
+
+1. Create an Ethernet interface on **A** for the trunk interface configured in
+   YAP (if it is not already created).
+
+
+.. tip::
+
+   You can find the configured trunk interface for **A** with the YAP command::
+
+    yap agg-show <agg ID>
+
+   Look for the *trunk* value.
+
+
+2. Create a VLAN device on aggregator **A** having the interface created in the
+   previous step as the trunk, and having the VLAN ID configured in YAP for
+   **S** in **R** as the ID.
+
+   Configure the interface to be associated with space **S**.
+
+.. tip::
+
+   You can find the configured VLAN ID for **S** in **R** with the following YAP
+   command::
+
+    yap space-show <S key>
+
+   Below *VLAN associations*, look for **R** followed by the VLAN ID.
+
+
+3. Add an address to the VLAN interface created in the previous step,
+   using the IP configured by YAP for **S** on **A**.
+
+.. tip::
+
+   You can find the configured IP for **S** on **A** with the following YAP
+   command::
+
+    yap subnet-get <S key> <R>
+
+    Below `Aggregators`, look for **A** followed by the IP.
+
+
+4. Create an OSPF protocol on aggregator **A** with the following configuration.
+   Anything not specified should be left to its default value in the form.
+
+    - Name: mm_<space key>
+    - Space: <space>
+    - Protocol: OSPF
+    - Enable: Off
+    - IPv4 import: All
+    - IPv4 export All
+    - Channel: IPv4
+    - Area:
+        - Area ID: 0.0.0.0
+        - Interface:
+            - Pattern: <name of VLAN created in step 2>
+
+   Click 'add area' to open the area form for configuring the Area ID,
+   and click 'add interface' to open the interface form to add the interface
+   pattern.
+
+.. warning::
+
+    If you do not set *Enabled* off, you may unintentionally affect private
+    WAN traffic prematurely.
+
+
+Migration
+=========================
+
+One the prepartion steps have been done for every aggregator carrying space
+traffic, the space is ready to be migrated to managed mesh.
+
+.. warning::
+
+    There will be a brief space outage during the migration.
+
+To actually perform the migration, three things must be done:
+
+#. Delete the space in YAP:
+    #. For each region, run *yap vlan-remove <space> <region>*
+    #. Run *yap space-delete <space>*
+#. Change the space mode from 'with private WAN routers' to 'managed mesh'
+#. Enable all the protocols created during the preparation phase.
+
+Confirm these protocols peer with any upstream neighbors in each region and that
+private WAN routes are being propogated.

salt/top.sls

@@ -1,2 +1,2 @@
-  'P@type:(aggregator|vxr)'
+  'P@type:(aggregator|vxr)':
     - yap

salt/yap/aggregator/config

@@ -1,6 +1,9 @@
 YAP_ID='{{ pillar['yap']['yap_id'] }}'
 VLAN_TRUNK='{{ pillar['yap']['trunk'] }}'
 declare -A SPACES
-{% if pillar['yap']['spaces'] %}{% for name, space in pillar['yap']['spaces'].items() %}
+{% if pillar['yap']['spaces'] %}
-SPACES[{{ name }}]='{{ space['id'] }} {{ space['vlan'] }}'
+{% for name, space in pillar['yap']['spaces'].items() -%}
-{% endfor %}{% endif %}
+{% if space.get('id') -%}
+SPACES[{{ name }}]='{{ space['id'] }} {{ space['vlan'] }} {{ space.get('ip', '') }}'
+{% endif %}
+{%- endfor %}{% endif %}

salt/yap/aggregator/init.sls

@@ -1,3 +1,4 @@
+{% if pillar.get('yap', None) %}
 /etc/bonding/bird/custom-external-bird.conf:
   file.managed:
     - source: salt://{{ tpldir }}/custom-external-bird.conf
@@ -14,17 +15,37 @@
 /etc/yap/bird:
   file.directory
 
+/etc/yap/spaces/bird:
+  file.directory:
+    - makedirs: true
+
 /etc/yap/config:
   file.managed:
     - source: salt://{{ tpldir }}/config
     - mode: 0640
     - template: jinja
 
+{% if pillar['yap']['spaces'] %}{% for name, space in pillar['yap']['spaces'].items() %}
+/etc/yap/spaces/bird/{{ name }}.conf:
+{% if space.get('bird_config', None) %}
+    file.managed:
+      - mode: 0644
+      - contents_pillar: yap:spaces:{{ name }}:bird_config
+{% else %}
+    file.absent
+{% endif %}
+{% endfor %}{% endif %}
+
 /etc/systemd/system/yap.service:
   file.managed:
     - source: salt://{{ tpldir }}/yap.service
     - mode: 0644
 
+/etc/systemd/system/yap-space@.service:
+  file.managed:
+    - source: salt://{{ tpldir }}/yap-space@.service
+    - mode: 0644
+
 /etc/systemd/system/yap-check-policy-rules.service:
   file.managed:
     - source: salt://{{ tpldir }}/yap-check-policy-rules.service
@@ -40,9 +61,12 @@ yap_service:
     - name: yap.service
     - enable: True
     - restart: True
+    - provider: systemd
 
 yap_check_policy_rules_timer:
   service.running:
     - name: yap-check-policy-rules.timer
     - enable: True
     - restart: True
+    - provider: systemd
+{% endif %}

salt/yap/aggregator/yap-check-policy-rules.service

@@ -1,5 +1,7 @@
 [Unit]
 Description=YAP policy rule checker
+BindsTo=yap.service
+After=yap.service
 
 [Service]
 Type=oneshot

salt/yap/aggregator/yap-space@.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=YAP space %i
+BindsTo=yap.service
+After=yap.service
+
+[Service]
+Type=simple
+RemainAfterExit=true
+ExecStartPre=/usr/local/bin/yap service-start %i
+ExecStart=/usr/local/bin/yap check-policy-rules %i
+ExecStop=/usr/local/bin/yap service-stop %i
+ExecReload=/usr/local/bin/yap reload
+Restart=on-failure
+RestartSec=1
+StartLimitInterval=1
+
+[Install]
+WantedBy=multi-user.target

salt/yap/aggregator/yap.service

@@ -1,12 +1,17 @@
 [Unit]
 Description=Yet Another Private WAN
-After=network.target
+BindsTo=node.service
+After=node.service
 
 [Service]
-Type=oneshot
+Type=simple
 RemainAfterExit=true
 ExecStart=/usr/local/bin/yap start-all
 ExecStop=/usr/local/bin/yap stop-all
+ExecReload=/usr/local/bin/yap reload
+Restart=on-failure
+RestartSec=1
+StartLimitInterval=1
 
 [Install]
 WantedBy=multi-user.target

salt/yap/aggregator/yap_aggregator

@@ -7,6 +7,9 @@ PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
 source /etc/yap/config
 
+WAIT_TIME=10  # time to wait for YAP lock, in seconds
+LOCKFILE=/tmp/yap.lockfile
+
 function check_args() {
     for arg_def in "$@" ; do
         OLD_IFS="$IFS"
@@ -85,10 +88,33 @@ get_links() {
 }
 
 
+function fail_lock() {
+    echo "Timed out waiting for exclusive lock on yap"
+    exit 1
+}
+
+
 function start() {
     args=$(check_args space,,$1) || return 1
     eval $args
 
+    echo "Starting $space"
+    if [ -z "${SPACES[$space]}" ] ; then
+        echo "Space not found"
+        return 1
+    fi
+
+    systemctl restart "yap-space@$space.service" --no-block
+}
+
+
+function service_start() {
+    args=$(check_args space,,$1) || return 1
+    eval $args
+
+    service_stop $space ||:
+
+    echo "Starting $space"
     if [ -z "${SPACES[$space]}" ] ; then
         echo "Space not found"
         return 1
@@ -97,13 +123,27 @@ function start() {
     set -- ${SPACES[$space]}
     space_id=$1
     vlan_id=$2
+    vlan_ip=$3
 
-    vlan_ip=$(get_vlan_ip $vlan_id $YAP_ID)
+    if [ -z $vlan_ip ] ; then
+        vlan_ip="$(get_vlan_ip $vlan_id $YAP_ID)/21"
+    fi
     table_id=$(get_table_id $space_id)
 
+    # Wait for bird to be up
+    while true; do
+        if pwanbirdc - show protocols | grep "krt${table_id}ipv4" &> /dev/null
+        then
+            break
+        else
+            sleep 1
+        fi
+    done
+
     # add VLAN
+    ip link set $VLAN_TRUNK up
     ip link add link $VLAN_TRUNK name vl-$space type vlan id $vlan_id
-    ip addr add $vlan_ip/21 dev vl-$space
+    ip addr add $vlan_ip dev vl-$space
     ip rule add iif vl-$space lookup $table_id prio 900
     ip link set vl-$space up
 
@@ -140,8 +180,12 @@ protocol ospf 'ospf_${space}' {
 EOF
     fi
 
-    pwanbirdc - configure soft
+    if [ -f "/etc/yap/spaces/bird/$space.conf" ] ; then
+        echo "include \"/etc/yap/spaces/bird/$space.conf\";" >> /etc/yap/bird/$space.conf
+    fi
 
+    reload
+    disable_bird_protocols $space
 }
 
 
@@ -149,6 +193,18 @@ function stop() {
     args=$(check_args space,,$1) || return 1
     eval $args
 
+    rm -f /etc/yap/bird/$space.conf
+
+    systemctl stop "yap-space@$space.service" --no-block
+}
+
+
+function service_stop() {
+    args=$(check_args space,,$1) || return 1
+    eval $args
+
+    rm -f /etc/yap/bird/$space.conf
+
     if [ -z "${SPACES[$space]}" ] ; then
         echo "Space not found"
         return 1
@@ -159,12 +215,10 @@ function stop() {
     vlan_id=$2
     table_id=$(get_table_id $space_id)
 
-    rm -f /etc/yap/bird/$space.conf
+    reload ||:
-    pwanbirdc - configure soft ||:
-
-    ip link del vl-$space ||:
-    ip rule del iif vl-$space lookup $table_id prio 900 ||:
 
+    ip link del vl-$space &>/dev/null ||:
+    ip rule del iif vl-$space lookup $table_id prio 900 &>/dev/null ||:
 }
 
 
@@ -190,26 +244,56 @@ function status() {
 
 
 function start_all() {
-    for name in "${!SPACES[@]}" ; do
+    (
-        start $name
+        flock -x -w $WAIT_TIME 200 || fail_lock
-    done
+        for name in "${!SPACES[@]}" ; do
+            restart $name
+        done
+    ) 200>$LOCKFILE
 }
 
 
 function stop_all() {
-    for name in "${!SPACES[@]}" ; do
+    (
-        stop $name
+        flock -x -w $WAIT_TIME 200 || fail_lock
-    done
+        for name in "${!SPACES[@]}" ; do
-    stop_unknown
+            stop $name
+        done
+        stop_unknown
+    ) 200>$LOCKFILE
+
+    # Catch any spaces that could be running which we don't know about
+    systemctl stop yap-space@*.service
+}
+
+
+function reload() {
+    pwanbirdc - configure soft
+}
+
+
+function restart() {
+    args=$(check_args space,,$1) || return 1
+    eval $args
+
+    rm -f /etc/yap/bird/$space.conf
+    if [ -z "${SPACES[$space]}" ] ; then
+        echo "Space not found"
+        return 1
+    fi
+
+    systemctl restart "yap-space@$space.service" --no-block
 }
 
 
 function restart_all() {
-    for name in "${!SPACES[@]}" ; do
+    (
-        stop $name
+        flock -x -w $WAIT_TIME 200 || fail_lock
-        start $name
+        for name in "${!SPACES[@]}" ; do
-    done
+            restart $name
-    stop_unknown
+        done
+        stop_unknown
+    ) 200>$LOCKFILE
 }
 
 
@@ -229,10 +313,7 @@ function stop_unknown() {
         fi
     done
 
-    ip rule | grep -e 'iif vl-' | sed -e 's/.* iif \(\w\+-\w\+\).*lookup \([0-9]\+\)/\1 \2/g' | while read $rule ; do
+    ip rule | grep -e 'iif vl-' | sed -e 's/.* iif \(\w\+-\w\+\).*lookup \([0-9]\+\)/\1 \2/g' | while read link table ; do
-        set -- $line
-        link=$1
-        table=$2
         name=${link:3}
         if [ -z "${SPACES[$name]}" ] ; then
             ip rule del from all iif $link lookup $table
@@ -241,19 +322,52 @@ function stop_unknown() {
 }
 
 
-function check_policy_rules() {
+function disable_bird_protocols() {
-    for space in "${!SPACES[@]}" ; do
+    args=$(check_args space,,$1) || return 1
-        set -- ${SPACES[$space]}
+    eval $args
-        space_id=$1
-        table_id=$(get_table_id $space_id)
 
-        if ! ip rule | grep -qe "iif vl-$space" ; then
+    bird_version=$(bird --version |& cut -d ' ' -f 3)
-            if ip link show dev vl-$space > /dev/null 2>&1 ; then
+    if [[ $bird_version =~ ^2 ]] ; then
-                echo "Adding missing ip rule for $space"
+        pwanbirdc - show protocols| grep -e "^spc${space}_pwr" | cut -d ' ' -f1 | xargs -r -l pwanbirdc - disable
-                ip rule add iif vl-$space lookup $table_id prio 900
+    else
-            fi
+        pwanbirdc $space show protocols | grep -e '^pwr' | cut -d ' ' -f1 | xargs -r -l pwanbirdc $space disable
+    fi
+}
+
+
+function _check_policy_rule() {
+    args=$(check_args space,,$1) || return 1
+    eval $args
+
+    set -- ${SPACES[$space]}
+    space_id=$1
+    table_id=$(get_table_id $space_id)
+
+    disable_bird_protocols $space
+
+    if ! ip rule | grep -qe "iif vl-$space" ; then
+        if ip link show dev vl-$space &> /dev/null ; then
+            echo "Adding missing ip rule for $space"
+            ip rule add iif vl-$space lookup $table_id prio 900
         fi
-    done
+    fi
+}
+
+
+function check_policy_rules() {
+    args=$(check_args space,skip,$1) || return 1
+    eval $args
+
+    if [ -z "$space" ] ; then
+        (
+            flock -x -w $WAIT_TIME 200 || fail_lock
+            for space in "${!SPACES[@]}" ; do
+                _check_policy_rule $space
+            done
+        ) 200>$LOCKFILE
+    else
+        _check_policy_rule $space
+    fi
 }
 
 
@@ -270,6 +384,7 @@ function usage() {
     echo "start-all"
     echo "stop-all"
     echo "restart-all"
+    echo "reload"
     echo "stop-unknown"
     echo "check-policy-rules"
     echo
@@ -290,8 +405,10 @@ case "$action" in
         stop $2
         ;;
     restart)
-        stop $2
+        restart $2
-        start $2
+        ;;
+    reload)
+        reload
         ;;
     status)
         status $2
@@ -308,8 +425,14 @@ case "$action" in
     stop-unknown)
         stop_unknown
         ;;
+    service-stop)
+        service_stop $2
+        ;;
+    service-start)
+        service_start $2
+        ;;
     check-policy-rules)
-        check_policy_rules
+        check_policy_rules $2
         ;;
     *)
         usage

salt/yap/vxr/bird.conf

@@ -22,3 +22,4 @@ protocol kernel {
 }
 
 include "/etc/yap/bird/*.conf";
+include "/etc/yap/bird_static/*.conf";

salt/yap/vxr/config

@@ -1,5 +1,6 @@
 YAP_ID='{{ pillar['yap']['yap_id'] }}'
 VLAN_TRUNK='{{ pillar['yap']['trunk'] }}'
+GLOBAL='{{ pillar['yap']['global'] }}'
 IPSEC_KEY='{{ pillar['yap'].get('ipsec_key', '') }}'
 ADMIN_HOSTS="74.121.32.0/22"
 {% if pillar['yap']['vxlan_peers'] %}
@@ -9,3 +10,8 @@ declare -A SPACES
 {% if pillar['yap']['spaces'] %}{% for name, space in pillar['yap']['spaces'].items() %}
 SPACES[{{ name }}]='{{ space['id'] }} {{ space['vlan'] }}'
 {% endfor %}{% endif %}
+
+declare -A GLOBAL_INTERFACE_OPTIONS
+{% if pillar['yap']['global_interface_options'] %}{% for name, value in pillar['yap']['global_interface_options'].items() %}
+GLOBAL_INTERFACE_OPTIONS[{{ name }}]='{{ value }}'
+{% endfor %}{% endif %}

salt/yap/vxr/init.sls

@@ -15,6 +15,10 @@ bird:
     - enable: True
     - restart: True
 
+iptables:
+  pkg.installed:
+    - refresh: false
+
 nftables:
   pkg.installed:
     - refresh: false
@@ -39,6 +43,9 @@ ipsec-tools:
 /etc/yap/bird:
   file.directory
 
+/etc/yap/bird_static:
+  file.directory
+
 /etc/yap/config:
   file.managed:
     - source: salt://{{ tpldir }}/config
@@ -66,3 +73,15 @@ yap_firewall_service:
     - name: yap-firewall
     - enable: True
     - restart: True
+
+yap_ipv4_forward:
+  sysctl.present:
+    - name: net.ipv4.ip_forward
+    - value: 1
+    - config: /etc/sysctl.d/yap.conf
+
+yap_ipv6_forward:
+  sysctl.present:
+    - name: net.ipv6.conf.all.forwarding
+    - value: 1
+    - config: /etc/sysctl.d/yap.conf

salt/yap/vxr/yap_vxr

@@ -148,6 +148,12 @@ function start_firewall() {
     for host in $VXLAN_PEERS ; do
         mesh_hosts="${mesh_hosts} ${host},"
     done
+    if [ ! -z "$admin_hosts" ] ; then
+        admin_hosts="ip saddr {$admin_hosts} jump input-admin"
+    fi
+    if [ ! -z "$mesh_hosts" ] ; then
+        mesh_hosts="ip saddr {$mesh_hosts} jump input-mesh"
+    fi
     tmp=$(mktemp)
     cat <<EOF > $tmp
 flush ruleset
@@ -172,14 +178,19 @@ table inet filter {
         ip6 nexthdr icmpv6 accept
 
         # Mesh hosts
-        ip saddr {$mesh_hosts} jump input-mesh
+        $mesh_hosts
 
-        # Backhaul
+        # igmp
-        iifname "vl-*" jump input-backhaul
+        ip protocol igmp accept
-        iifname "vx-*" jump input-backhaul
+
+        # ospf
+        ip protocol ospfigp accept
+
+        # iperf
+        tcp dport 5201 accept
 
         # Allow administrative hosts
-        ip saddr {$admin_hosts} jump input-admin
+        $admin_hosts
 
         # Reject
         reject with icmpx type admin-prohibited
@@ -197,19 +208,6 @@ table inet filter {
         # VXLAN
         udp dport 4789 accept
     }
-
-    # Backhaul rules
-    #
-    chain input-backhaul {
-        # igmp
-        ip protocol igmp accept
-
-        # ospf
-        ip protocol ospfigp accept
-
-        # iperf
-        tcp dport 5201 accept
-    }
 }
 
 EOF
@@ -240,6 +238,7 @@ function start() {
     table_id=$(get_table_id $space_id)
 
     # add VLAN
+    ip link set $VLAN_TRUNK up
     ip link add link $VLAN_TRUNK name vl-$space type vlan id $vlan_id
     ip addr add $vlan_ip/21 dev vl-$space
     ip rule add iif vl-$space lookup $table_id prio 1000
@@ -342,6 +341,73 @@ function status() {
 }
 
 
+function start_global() {
+    if [ "$GLOBAL" != "True" ] ; then
+        return
+    fi
+
+    vxlan_ip=$(get_vxlan_ip 0 $YAP_ID)
+    router_id=$(ip -o r get to 1.1.1.1 | sed -e 's/.*src \([0-9.]*\).*/\1/')
+    global_interface=$(ip -o r get to 1.1.1.1 | sed -e 's/.*dev \([a-z0-9.-]*\).*/\1/')
+
+    # add VXLAN
+    ip link add global type vxlan id 0 dstport 4789
+    ip link set global mtu 1432
+    ip addr add $vxlan_ip/21 dev global
+    ip link set global up
+
+    if [ ! -z "$VXLAN_PEERS" ] ; then
+        for peer in $VXLAN_PEERS ; do
+            bridge fdb append to 00:00:00:00:00:00 dst $peer dev global
+        done
+    fi
+
+    cat <<EOF > /etc/yap/bird/__global.conf
+protocol ospf global_ospf {
+    router id $router_id;
+    area 0.0.0.0 {
+        interface "${global_interface}" {
+            cost 10;
+EOF
+
+    for option in "${!GLOBAL_INTERFACE_OPTIONS[@]}" ; do
+        echo -e "            $option ${GLOBAL_INTERFACE_OPTIONS[$option]};" >> /etc/yap/bird/__global.conf
+    done
+
+cat <<EOF >> /etc/yap/bird/__global.conf
+        };
+        interface "global" {
+            cost 100;
+        };
+    };
+    ipv4 {
+        preference 1000;
+        import all;
+        export all;
+    };
+}
+EOF
+
+    birdc configure soft
+}
+
+
+function stop_global() {
+    rm -f /etc/yap/bird/__global.conf
+    birdc configure soft
+
+    ip link del global ||:
+}
+
+
+function status_global() {
+    birdc "show protocol global_ospf" | tail -n+3
+    echo
+    ip route show
+    echo
+}
+
+
 function start_all() {
     start_ipsec
     for name in "${!SPACES[@]}" ; do
@@ -384,10 +450,7 @@ function stop_unknown() {
         fi
     done
 
-    ip rule | grep -e 'iif v[lx]-' | sed -e 's/.* iif \(\w\+-\w\+\).*lookup \([0-9]\+\)/\1 \2/g' | while read $rule ; do
+    ip rule | grep -e 'iif v[lx]-' | sed -e 's/.* iif \(\w\+-\w\+\).*lookup \([0-9]\+\)/\1 \2/g' | while read link table ; do
-        set -- $line
-        link=$1
-        table=$2
         name=${link:3}
         if [ -z "${SPACES[$name]}" ] ; then
             ip rule del from all iif $link lookup $table
@@ -406,6 +469,11 @@ function usage() {
     echo "restart <space>"
     echo "status <space>"
     echo
+    echo "start-global"
+    echo "stop-global"
+    echo "restart-global"
+    echo "status-global"
+    echo
     echo "start-all"
     echo "stop-all"
     echo "restart-all"
@@ -440,6 +508,19 @@ case "$action" in
     status)
         status $2
         ;;
+    start-global)
+        start_global
+        ;;
+    stop-global)
+        stop_global
+        ;;
+    restart-global)
+        stop_global
+        start_global
+        ;;
+    status-global)
+        status_global
+        ;;
     start-all)
         start_all
         ;;

update-docs.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+RADOSGW_USER="yap-docs"
+BUCKET_NAME="yap-docs"
+BUCKET="s3://$BUCKET_NAME"
+ACCESS_KEY=$(radosgw-admin user info --uid=$RADOSGW_USER | grep "access_key" | cut -d '"' -f 4)
+SECRET_ACCESS_KEY=$(radosgw-admin user info --uid=$RADOSGW_USER | grep "secret_key" | cut -d '"' -f 4)
+
+S3CMD_OPTIONS="--access_key=$ACCESS_KEY --secret_key=$SECRET_ACCESS_KEY --no-ssl --acl-public --host=rgw.multapplied.net --host-bucket='yap-docs.rgw.multapplied.net'"
+echo "$S3CMD_OPTIONS"
+S3CMD="s3cmd $S3CMD_OPTIONS"
+
+$S3CMD mb $BUCKET
+$S3CMD ws-create $BUCKET
+$S3CMD sync docs/ $BUCKET

yap

@@ -75,6 +75,20 @@ function set_var() {
 }
 
 
+# set_var <var> <value>
+#
+# Set variable in data store
+#
+function set_var_from_file() {
+    varfile="${DATA_DIR}/$1"
+    vardir=$(dirname "$varfile")
+    if [ ! -d "$vardir" ] ; then
+        install -d -m 0755 "$vardir"
+    fi
+    cp "$2" "$varfile"
+}
+
+
 # del_var <var>
 #
 # Delete variable from data store
@@ -167,12 +181,58 @@ function get_section_path() {
 function list_sections() {
     if [ -d $DATA_DIR/$1 ] ; then
         for var in $(find $DATA_DIR/$1 -mindepth 1 -maxdepth 1 -type d | sort) ; do
-            echo ${var#$DATA_DIR}
+            echo ${var#$DATA_DIR/}
         done
     fi
 }
 
 
+function ip2dec() {
+    local a b c d ip=$@
+    IFS=. read -r a b c d <<< "$ip"
+    printf '%d\n' "$((a * 256 ** 3 + b * 256 ** 2 + c * 256 + d))"
+}
+
+
+function dec2ip() {
+    local delim ip dec=$@
+    for e in {3..0} ; do
+        ((octet = dec / (256 ** e) ))
+        ((dec -= octet * 256 ** e))
+        ip+=$delim$octet
+        delim=.
+    done
+    printf '%s' "$ip"
+}
+
+
+# Generate an IP for a VLAN interface based on the VLAN and and the YAP ID.
+# The first octet will always be 100, while the other 3 octets are split into
+# 3 sections of varying size to contain the IP type, the VLAN, and the YAP ID:
+#
+#  type: 1-bit
+#  vlan ID: 12-bits
+#  YAP ID: 11-bits
+#
+# The resulting IP should be used with a prefix length of 21
+#
+function get_vlan_ip() {
+    vlan_id=$1
+    yap_id=$2
+
+    # Start at 100.0.0.0
+    local ip=$(ip2dec 100.0.0.0)
+
+    # Add the VLAN ID, shifted 11-bits
+    ((ip += vlan_id << 11))
+
+    # Add the YAP ID
+    ((ip += yap_id))
+
+    dec2ip $ip
+}
+
+
 function validate_ip() {
     if ! [[ $1 =~ ^(0*(1?[0-9]{1,2}|2([0-4][0-9]|5[0-5]))\.){3}0*(1?[0-9]{1,2}|2([‌​0-4][0-9]|5[0-5]))$ ]] ; then
         return 1
@@ -353,12 +413,18 @@ function salt_update() {
         local region=$(get_var $vxr/region)
         local trunk=$(get_var $vxr/trunk)
         local yap_id=$(get_var $vxr/yap_id)
+        local global=$(get_var $vxr/global false)
         local name=$(basename $vxr)
         local vxr_peers
         {
             echo -e "yap:"
             echo -e "  yap_id: $yap_id"
             echo -e "  trunk: $trunk"
+            echo -e "  global: $global"
+            echo -e "  global_interface_options:"
+            for option in $(list_vars $vxr/global_interface_options) ; do
+                echo -e "    '$(basename $option)': '$(get_var $option)'"
+            done
             echo -e "  vxlan_peers:"
             for peer_vxr in $(list_sections /vxrs) ; do
                 if [ "$peer_vxr" != "$vxr" ] ; then
@@ -389,6 +455,31 @@ function salt_update() {
             echo -e "  yap_id: $yap_id"
             echo -e "  trunk: $trunk"
         } > $new_host_file
+        new_host_spaces_file=$(mktemp)
+        has_spaces=0
+        {
+            echo -e "  spaces:"
+            for space in $(list_sections $aggregator/spaces) ; do
+                has_spaces=1
+                local space_name=$(basename $space)
+                {
+                    echo -e "    $space_name:"
+                    local ip=$(get_var $space/ip)
+                    if [ -n "$ip" ] ; then
+                        echo -e "      ip: $ip"
+                    fi
+                    local bird_config=$(get_var $space/bird_config)
+                    if [ -n "$bird_config" ] ; then
+                        echo -e "      bird_config: |-"
+                        echo -e "$bird_config" | sed -e 's/^/        /g'
+                    fi
+                }
+            done
+        } > $new_host_spaces_file
+
+        if [ $has_spaces = 1 ] ; then
+            cat $new_host_spaces_file >> $new_host_file
+        fi
         mv $new_host_file $SALT_PILLARS/yap/hosts/node-$id.sls
         chmod 0644 $SALT_PILLARS/yap/hosts/node-$id.sls
 
@@ -415,7 +506,21 @@ function salt_exec() {
     nodelist="$1"
     shift
 
-    salt -C "$(salt_nodelist $nodelist)" $@
+    salt -C "$nodelist" "$@"
+}
+
+
+function upgrade() {
+    args=$(check_args region,skip,$1)
+    eval $args
+
+    if [ ! -z "$region" ] ; then
+        salt_update "$(get_region_nodelist $region)"
+        salt_exec "$(get_region_nodelist $region)" service.restart yap ||:
+    else
+        salt_update "$(get_full_nodelist)"
+        salt_exec "$(get_full_nodelist)" service.restart yap ||:
+    fi
 }
 
 
@@ -453,9 +558,10 @@ function region_show() {
     echo "VLAN associations:"
 
     for space in $(list_sections "/regions/$region/spaces") ; do
-        var="/regions/$region/spaces/$space/vlan"
+        var="$space/vlan"
         if has_var "$var" ; then
-            echo "    $(basename space) $(get_var $var)"
+            vlan_id=$(get_var $var)
+            echo "    $(basename space) $vlan_id: $(get_vlan_ip $vlan_id 0)/21"
         fi
     done
 
@@ -535,7 +641,13 @@ function space_add() {
         return 1
     fi
 
-    id=$(api_get /api/v3/spaces/$space/ | jq .id)
+    space_json=$(api_get /api/v3/spaces/$space/)
+    private_wan_enabled=$(echo $space_json| jq .private_wan_enabled)
+    if ! $private_wan_enabled ; then
+        echo "Space ${space} does not have private WAN enabled."
+        return 1
+    fi
+    id=$(echo $space_json| jq .id)
     set_var "/spaces/$space/id" "$id"
 
     space_show $space
@@ -546,15 +658,66 @@ function space_delete() {
     args=$(check_args space,,$1) || return 1
     eval $args
 
-    del_section "/spaces/$space"
+    for section in $(list_sections /regions/); do
-
+        for space_path in $(list_sections $section/spaces/); do
-    for section in $(list_sections /regions/spaces/) ; do
+            if [ "$(basename $space_path)" = "$space" ] ; then
-        if [ "$(basename $section)" = "$space" ] ; then
+                echo "You must remove the VLAN for region $(basename $section) first. Use vlan-remove."
-            del_section "$section"
+                return 1
-        fi
+            fi
+        done
     done
 
-    salt_update
+    # Clean up aggregator's space IP/custom configuration automatically
+    for agg_section in $(list_sections /aggregators/); do
+        for space_path in $(list_sections $agg_section/spaces/); do
+            if [ "$(basename $space_path)" = "$space" ] ; then
+                del_section "$space_path"
+            fi
+        done
+    done
+
+    del_section "/spaces/$space"
+}
+
+
+#
+# Subnet commands
+#
+
+function subnet_get() {
+    args=$(check_args space,,$1, region,,$2, vlan_id,skip,$3) || return 1
+    eval $args
+
+    if ! has_section "/spaces/$space" ; then
+        echo "Space does not exist"
+        return 1
+    fi
+
+    if ! has_section "/regions/$region" ; then
+        echo "Region does not exist"
+        return 1
+    fi
+
+    if ! has_var "/regions/$region/spaces/$space/vlan"; then
+        if [ -z "$vlan_id" ] ; then
+            echo "No VLAN configured: argument 'vlan_id' is required"
+            return 1
+        fi
+        vlan_validate $space $region $vlan_id
+    else
+        vlan_id=$(get_var "/regions/$region/spaces/$space/vlan")
+    fi
+
+    echo "Subnet: $(get_vlan_ip $vlan_id 0)/21"
+    echo "Firewall: $(get_vlan_ip $vlan_id 1)"
+    echo "Aggregators:"
+
+    for aggregator in $(list_sections "/aggregators") ; do
+        if [ "$(get_var $aggregator/region)" = $region ] ; then
+            ip=$(get_vlan_ip $vlan_id $(get_var $aggregator/yap_id))
+            echo "    $(get_var $aggregator/name): $ip"
+        fi
+    done
 }
 
 
@@ -635,6 +798,93 @@ function aggregator_trunk_set() {
 }
 
 
+function aggregator_set_space_ip() {
+    args=$(check_args aggregator,,$1 space,,$2 ip,,$3) || return 1
+    eval $args
+
+    if ! has_section "/aggregators/$aggregator" ; then
+        echo "Aggregator does not exist"
+        return 1
+    fi
+
+    if ! has_section "/spaces/$space" ; then
+        echo "Space does not exist"
+        return 1
+    fi
+
+    set_var "/aggregators/$aggregator/spaces/$space/ip" "$ip"
+
+    salt_update node-$aggregator ||:
+    salt_exec node-$aggregator cmd.run "yap restart $space" ||:
+
+    aggregator_show $aggregator
+}
+
+
+function aggregator_remove_space_ip() {
+    args=$(check_args aggregator,,$1 space,,$2) || return 1
+    eval $args
+
+    if ! has_section "/aggregators/$aggregator" ; then
+        echo "Aggregator does not exist"
+        return 1
+    fi
+
+    del_var "/aggregators/$aggregator/spaces/$space/ip"
+
+    salt_update node-$aggregator ||:
+    salt_exec node-$aggregator cmd.run "yap restart $space" ||:
+
+    aggregator_show $aggregator
+}
+
+
+function aggregator_set_space_bird_config() {
+    args=$(check_args aggregator,,$1 space,,$2 file,,$3) || return 1
+    eval $args
+
+    if ! has_section "/aggregators/$aggregator" ; then
+        echo "Aggregator does not exist"
+        return 1
+    fi
+
+    if ! has_section "/spaces/$space" ; then
+        echo "Space does not exist"
+        return 1
+    fi
+
+    if [ ! -f $file ] ; then
+        echo "File does not exist"
+        return 1
+    fi
+
+    set_var_from_file "/aggregators/$aggregator/spaces/$space/bird_config" $file
+
+    salt_update node-$aggregator ||:
+    salt_exec node-$aggregator cmd.run "yap restart $space" ||:
+
+    aggregator_show $aggregator
+}
+
+
+function aggregator_remove_space_bird_config() {
+    args=$(check_args aggregator,,$1 space,,$2) || return 1
+    eval $args
+
+    if ! has_section "/aggregators/$aggregator" ; then
+        echo "Aggregator does not exist"
+        return 1
+    fi
+
+    del_var "/aggregators/$aggregator/spaces/$space/bird_config"
+
+    salt_update node-$aggregator ||:
+    salt_exec node-$aggregator cmd.run "yap restart $space" ||:
+
+    aggregator_show $aggregator
+}
+
+
 function aggregator_delete() {
     args=$(check_args aggregator,,$1) || return 1
     eval $args
@@ -680,6 +930,7 @@ function vxr_show() {
     echo "ip: $(get_var /vxrs/$vxr/ip)"
     echo "region: $(get_var /vxrs/$vxr/region)"
     echo "trunk: $(get_var /vxrs/$vxr/trunk)"
+    echo "global: $(get_var /vxrs/$vxr/global false)"
 }
 
 
@@ -708,11 +959,12 @@ function vxr_add() {
     set_var "/vxrs/$vxr/yap_id" "$(get_next_yap_ip)"
 
     # All VXRs need the IP
-    salt_update $(get_vxr_nodelist)
+    salt_update $(get_vxr_nodelist) ||:
 
     # Additional apply for authorized keys, etc.
     salt $vxr state.apply ||:
 
+    salt_exec $(get_vxr_nodelist) service.restart yap-firewall ||:
     salt_exec $(get_vxr_nodelist) service.restart yap ||:
 
     vxr_show $vxr
@@ -760,6 +1012,78 @@ function vxr_ip_set() {
 }
 
 
+function vxr_enable_global() {
+    args=$(check_args vxr,,$1) || return 1
+    eval $args
+
+    if ! has_section "/vxrs/$vxr" ; then
+        echo "VXR does not exist"
+        return 1
+    fi
+
+    set_var "/vxrs/$vxr/global" true
+
+    salt_update $(get_vxr_nodelist) ||:
+    salt_exec $vxr cmd.run 'yap restart-global' ||:
+
+    vxr_show $vxr
+}
+
+
+function vxr_disable_global() {
+    args=$(check_args vxr,,$1) || return 1
+    eval $args
+
+    if ! has_section "/vxrs/$vxr" ; then
+        echo "VXR does not exist"
+        return 1
+    fi
+
+    set_var "/vxrs/$vxr/global" false
+
+    salt_update $(get_vxr_nodelist) ||:
+    salt_exec $vxr cmd.run 'yap restart-global' ||:
+
+    vxr_show $vxr
+}
+
+
+function vxr_set_global_interface_option() {
+    args=$(check_args vxr,,$1 name,,$2 value,,$3) || return 1
+    eval $args
+
+    if ! has_section "/vxrs/$vxr" ; then
+        echo "VXR does not exist"
+        return 1
+    fi
+
+    set_var "/vxrs/$vxr/global_interface_options/$name" "$value"
+
+    salt_update $(get_vxr_nodelist) ||:
+    salt_exec $vxr cmd.run 'yap restart-global' ||:
+
+    vxr_show $vxr
+}
+
+
+function vxr_delete_global_interface_option() {
+    args=$(check_args vxr,,$1 name,,$2) || return 1
+    eval $args
+
+    if ! has_section "/vxrs/$vxr" ; then
+        echo "VXR does not exist"
+        return 1
+    fi
+
+    del_var "/vxrs/$vxr/global_interface_options/$name"
+
+    salt_update $(get_vxr_nodelist) ||:
+    salt_exec $vxr cmd.run 'yap restart-global' ||:
+
+    vxr_show $vxr
+}
+
+
 function vxr_delete() {
     args=$(check_args vxr,,$1) || return 1
     eval $args
@@ -782,20 +1106,10 @@ function vxr_delete() {
 # VLAN commands
 #
 
-function vlan_set() {
+function vlan_validate() {
     args=$(check_args space,,$1 region,,$2 vlan_id,,$3) || return 1
     eval $args
 
-    if ! has_section /spaces/$space ; then
-        echo "Space does not exist"
-        return 1
-    fi
-
-    if ! has_section /regions/$region ; then
-        echo "Region does not exist"
-        return 1
-    fi
-
     if ! [[ $vlan_id =~ [0-9]* ]] ; then
         echo "VLAN must be numeric"
         return 1
@@ -811,10 +1125,34 @@ function vlan_set() {
         return 1
     fi
 
+    for space_path in $(list_sections /regions/$region/spaces); do
+        if [ "$(get_var $space_path/vlan)" = "$vlan_id" ] ; then
+            echo "VLAN ${vlan_id} conflicts with space $(basename $space_path) in region ${region}"
+            return 1
+        fi
+    done
+}
+
+function vlan_set() {
+    args=$(check_args space,,$1 region,,$2 vlan_id,,$3) || return 1
+    eval $args
+
+    if ! has_section /spaces/$space ; then
+        echo "Space does not exist"
+        return 1
+    fi
+
+    if ! has_section /regions/$region ; then
+        echo "Region does not exist"
+        return 1
+    fi
+
+    vlan_validate $space $region $vlan_id
+
     set_var "/regions/$region/spaces/$space/vlan" "$vlan_id"
 
     salt_update $(get_region_nodelist $region) ||:
-    salt_exec $(get_region_nodelist $region) cmd.run yap restart $space
+    salt_exec "$(get_region_nodelist $region)" cmd.run "yap restart $space"
 }
 
 
@@ -824,7 +1162,7 @@ function vlan_remove() {
 
     del_var "/regions/$region/spaces/$space/vlan"
 
-    salt_exec $(get_region_nodelist $region) cmd.run yap stop $space ||:
+    salt_exec "$(get_region_nodelist $region)" cmd.run "yap stop $space" ||:
     salt_update $(get_region_nodelist $region)
 }
 
@@ -845,7 +1183,7 @@ function ipsec_enable() {
     nodelist="$(get_vxr_nodelist)"
     if [ -n "$nodelist" ] ; then
         salt_update $nodelist ||:
-        salt_exec "$nodelist" cmd.run yap start-ipsec
+        salt_exec "$nodelist" cmd.run "yap start-ipsec"
     fi
 }
 
@@ -854,7 +1192,7 @@ function ipsec_disable() {
     nodelist="$(get_vxr_nodelist)"
     if [ -n "$nodelist" ] ; then
         salt_update $nodelist ||:
-        salt_exec "$nodelist" cmd.run yap stop-ipsec
+        salt_exec "$nodelist" cmd.run "yap stop-ipsec"
     fi
 }
 
@@ -874,17 +1212,27 @@ function usage() {
     echo "space-add <spacekey>"
     echo "space-delete <spacekey>"
     echo
+    echo "subnet-get <spacekey> <region> [vlan_id]"
+    echo
     echo "agg-list"
     echo "agg-show <id>"
     echo "agg-add <id> <region> <trunk>"
     echo "agg-trunk-set <id> <trunk>"
     echo "agg-delete <id>"
+    echo "agg-set-space-ip <id> <spacekey> <ip>"
+    echo "agg-remove-space-ip <id> <spacekey>"
+    echo "agg-set-space-bird-config <id> <spacekey> <filename>"
+    echo "agg-remove-space-bird-config <id> <spacekey>"
     echo
     echo "vxr-list"
     echo "vxr-show <name>"
     echo "vxr-add <name> <ip> <region> <trunk>"
     echo "vxr-ip-set <name> <ip>"
     echo "vxr-trunk-set <name> <trunk>"
+    echo "vxr-enable-global <name>"
+    echo "vxr-disable-global <name>"
+    echo "vxr-set-global-interface-option <name> <value>"
+    echo "vxr-delete-global-interface-option <name>"
     echo "vxr-delete <name>"
     echo
     echo "vlan-set <spacekey> <region> <vlan_id>"
@@ -895,6 +1243,7 @@ function usage() {
 
     echo "auth-set <email> <password>"
     echo "dump-config"
+    echo "upgrade [region]"
     echo
 }
 
@@ -914,6 +1263,9 @@ case "$action" in
     dump-config)
         dump_vars "$@"
         ;;
+    upgrade)
+        upgrade "$@"
+        ;;
     region-list)
         region_list "$@"
         ;;
@@ -938,6 +1290,9 @@ case "$action" in
     space-delete)
         space_delete "$@"
         ;;
+    subnet-get)
+        subnet_get "$@"
+        ;;
     agg-list|aggregator-list)
         aggregator_list "$@"
         ;;
@@ -953,6 +1308,18 @@ case "$action" in
     agg-delete|aggregator-delete)
         aggregator_delete "$@"
         ;;
+    agg-set-space-ip|aggregator-set-space-ip)
+        aggregator_set_space_ip  "$@"
+        ;;
+    agg-remove-space-ip|aggregator-remove-space-ip)
+        aggregator_remove_space_ip  "$@"
+        ;;
+    agg-set-space-bird-config|aggregator-set-space-bird-config)
+        aggregator_set_space_bird_config  "$@"
+        ;;
+    agg-remove-space-bird-config|aggregator-remove-space-bird-config)
+        aggregator_remove_space_bird_config  "$@"
+        ;;
     vxr-list)
         vxr_list "$@"
         ;;
@@ -962,9 +1329,21 @@ case "$action" in
     vxr-add)
         vxr_add "$@"
         ;;
-    set|vxr-trunk-set)
+    vxr-trunk-set)
         vxr_trunk_set "$@"
         ;;
+    vxr-enable-global)
+        vxr_enable_global "$@"
+        ;;
+    vxr-disable-global)
+        vxr_disable_global "$@"
+        ;;
+    vxr-set-global-interface-option)
+        vxr_set_global_interface_option "$@"
+        ;;
+    vxr-delete-global-interface-option)
+        vxr_delete_global_interface_option "$@"
+        ;;
     vxr-delete)
         vxr_delete "$@"
         ;;