#!/bin/bash

#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# devstack/plugin.sh
# Functions to control the configuration and operation of the OVN service

# Dependencies:
#
# ``functions`` file
# ``DEST`` must be defined
# ``STACK_USER`` must be defined

# ``stack.sh`` calls the entry points in this order:
#
# - install_ovn
# - configure_ovn
# - configure_ovn_plugin
# - init_ovn
# - start_ovn
# - stop_ovn
# - cleanup_ovn

# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace

# Defaults
# --------

# The git repo to use
OVN_REPO=${OVN_REPO:-https://github.com/openvswitch/ovs.git}
OVN_REPO_NAME=$(basename ${OVN_REPO} | cut -f1 -d'.')

# The project directory
NETWORKING_OVN_DIR=$DEST/networking-ovn

# The branch to use from $OVN_REPO
OVN_BRANCH=${OVN_BRANCH:-master}

# How to connect to ovsdb-server hosting the OVN SB database.
OVN_SB_REMOTE=${OVN_SB_REMOTE:-tcp:$HOST_IP:6642}

# How to connect to ovsdb-server hosting the OVN NB database
OVN_NB_REMOTE=${OVN_NB_REMOTE:-tcp:$HOST_IP:6641}

# A UUID to uniquely identify this system.  If one is not specified, a random
# one will be generated.  A randomly generated UUID will be saved in a file
# 'ovn-uuid' so that the same one will be re-used if you re-run DevStack.
OVN_UUID=${OVN_UUID:-}

# Whether or not to build the openvswitch kernel module from ovs.  This is required
# unless the distro kernel includes ovs+conntrack support.
OVN_BUILD_MODULES=$(trueorfalse True OVN_BUILD_MODULES)

# Whether or not to install the ovs python module from ovs source.  This can be
# used to test and validate new ovs python features.  This should only be used
# for development purposes since the ovs python version is controlled by OpenStack
# requirements.
OVN_INSTALL_OVS_PYTHON_MODULE=$(trueorfalse False OVN_INSTALL_OVS_PYTHON_MODULE)

# GENEVE overlay protocol overhead. Defaults to 38 bytes plus the IP version
# overhead (20 bytes for IPv4 (default) or 40 bytes for IPv6) which is determined
# based on the ML2 overlay_ip_version option. The ML2 framework will use this to
# configure the DHCP agent MTU option.
OVN_GENEVE_OVERHEAD=${OVN_GENEVE_OVERHEAD:-38}

# If using OVN_L3_MODE, this sets whether to create a public network and bridge.
# If set to True, a public network and subnet(s) will be created, and a router
# will be created to route the default private network to the public one.
# Can only be set to True if OVN_L3_MODE is being used (and not q-l3) and
# NEUTRON_CREATE_INITIAL_NETWORKS is True (the default).  There are known issues
# setting this to true in a multinode devstack setup
OVN_L3_CREATE_PUBLIC_NETWORK=$(trueorfalse False OVN_L3_CREATE_PUBLIC_NETWORK)

# ml2/config for neutron_sync_mode
OVN_NEUTRON_SYNC_MODE=${OVN_NEUTRON_SYNC_MODE:-log}

# If using OVN_L3_MODE, the type of OVN L3 Scheduler to use. The OVN L3
# Scheduler determines the hypervisor/chassis where a routers gateway should
# be hosted in OVN. The default OVN L3 scheduler is leastloaded
OVN_L3_SCHEDULER=${OVN_L3_SCHEDULER:-leastloaded}

# Neutron directory
NEUTRON_DIR=$DEST/neutron

# Set variables for building OVS from source
OVS_REPO=$OVN_REPO
OVS_REPO_NAME=$OVN_REPO_NAME
OVS_BRANCH=$OVN_BRANCH

# Utility Functions
# -----------------

# There are some ovs functions OVN depends on that must be sourced from
# the ovs neutron plugins. After doing this, the OVN overrides must be
# re-sourced.
source $TOP_DIR/lib/neutron_plugins/ovs_base
source $TOP_DIR/lib/neutron_plugins/openvswitch_agent
source $NETWORKING_OVN_DIR/devstack/override-defaults
source $NETWORKING_OVN_DIR/devstack/network_utils.sh

function is_ovn_service_enabled {
    ovn_service=$1
    is_service_enabled ovn && return 0
    is_service_enabled $ovn_service && return 0
    return 1
}

# NOTE(rtheis): Function copied from DevStack _neutron_ovs_base_setup_bridge
# and _neutron_ovs_base_add_bridge with the call to neutron-ovs-cleanup
# removed. The call is not relevant for OVN, as it is specific to the use
# of Neutron's OVS agent and hangs when running stack.sh because
# neutron-ovs-cleanup uses the OVSDB native interface.
function ovn_base_setup_bridge {
    local bridge=$1
    local addbr_cmd="sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge"

    if [ "$OVS_DATAPATH_TYPE" != "system" ] ; then
        addbr_cmd="$addbr_cmd -- set Bridge $bridge datapath_type=${OVS_DATAPATH_TYPE}"
    fi

    $addbr_cmd
    sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
}


# Entry Points
# ------------

# cleanup_ovn() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_ovn {
    local _pwd=$(pwd)
    cd $DEST/$OVN_REPO_NAME
    sudo make uninstall
    cd $_pwd
}

# configure_ovn() - Set config files, create data dirs, etc
function configure_ovn {
    echo "Configuring OVN"

    if [ -z "$OVN_UUID" ] ; then
        if [ -f ./ovn-uuid ] ; then
            OVN_UUID=$(cat ovn-uuid)
        else
            OVN_UUID=$(uuidgen)
            echo $OVN_UUID > ovn-uuid
        fi
    fi
}

function configure_ovn_plugin {
    echo "Configuring Neutron for OVN"

    if is_service_enabled q-svc ; then
        # NOTE(arosen) needed for tempest
        export NETWORK_API_EXTENSIONS=$(python -c \
            'from networking_ovn.common import extensions ;\
             print ",".join(extensions.ML2_SUPPORTED_API_EXTENSIONS)')
        if [[ "$OVN_L3_MODE" == "True" ]]; then
            export NETWORK_API_EXTENSIONS=$NETWORK_API_EXTENSIONS,$(python -c \
                'from networking_ovn.common import extensions ;\
                 print ",".join(extensions.ML2_SUPPORTED_API_EXTENSIONS_OVN_L3)')
        else
            export NETWORK_API_EXTENSIONS=$NETWORK_API_EXTENSIONS,$(python -c \
                'from networking_ovn.common import extensions ;\
                 print ",".join(extensions.ML2_SUPPORTED_API_EXTENSIONS_NEUTRON_L3)')
        fi
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ml2_type_geneve max_header_size=$OVN_GENEVE_OVERHEAD
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_nb_connection="$OVN_NB_REMOTE"
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_sb_connection="$OVN_SB_REMOTE"
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_l3_mode="$OVN_L3_MODE"
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn neutron_sync_mode="$OVN_NEUTRON_SYNC_MODE"
        if [[ "$OVN_L3_MODE" == "True" ]]; then
            populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_l3_scheduler="$OVN_L3_SCHEDULER"
        fi
        populate_ml2_config /$Q_PLUGIN_CONF_FILE ovn ovn_native_dhcp="$OVN_NATIVE_DHCP"
        populate_ml2_config /$Q_PLUGIN_CONF_FILE securitygroup enable_security_group="$Q_USE_SECGROUP"
        inicomment /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver
    fi

    # NOTE(rtheis): Conventional and native DHCP are not supported at the
    # same time since there aren't any confirmed benefits to supporting
    # both. Also, there are a couple problems with such a setup:
    #   1) Neutron still creates DHCP provisioning blocks when launching
    #      instances. As a result, the conventional DHCP agent must be
    #      up and remove the DHCP provisioning blocks or instances will
    #      fail to launch.
    #   2) Even though the conventional DHCP agent can setup a metadata
    #      proxy for isolated networks, host routes are not added to an
    #      instance. As a result, instances aren't able to access the proxy
    #      via "curl http://169.254.169.254".
    if is_service_enabled q-dhcp ; then
        if [[ "$OVN_NATIVE_DHCP" == "True" ]]; then
            die $LINENO "The q-dhcp service must be disabled with OVN_NATIVE_DHCP set to True."
        fi
    fi

    if is_service_enabled q-l3 ; then
        if [[ "$OVN_L3_MODE" == "True" ]]; then
            die $LINENO "The q-l3 service must be disabled with OVN_L3_MODE set to True."
        fi
    fi

    # NOTE(rtheis): OVN native layer-3 and DHCP services currently lack support
    # for metadata. Thus, enabling both native services also requires enabling
    # config drive to provide instances with metadata. However, if you choose the
    # conventional DHCP agent instead of the native DHCP service, you can
    # configure it to provide instances with metadata.
    if is_service_enabled q-dhcp ; then
        # Conventional DHCP agent must provide all metadata when OVN native
        # layer-3 is enabled. So for consistency, regardless of the layer-3
        # support, the conventional DHCP agent will be forced to provide
        # metadata for all networks.
        iniset $Q_DHCP_CONF_FILE DEFAULT force_metadata True
    elif [[ "$OVN_NATIVE_DHCP" == "True" && "$OVN_L3_MODE" == "True" ]]; then
        if is_service_enabled n-cpu ; then
            iniset $NOVA_CONF DEFAULT force_config_drive True
        fi
    fi

    if is_service_enabled q-qos ; then
        NEUTRON_CONF=/etc/neutron/neutron.conf
        iniset $NEUTRON_CONF qos notification_drivers ovn-qos
    fi
}

# init_ovn() - Initialize databases, etc.
function init_ovn {
    # clean up from previous (possibly aborted) runs
    # create required data files

    # Assumption: this is a dedicated test system and there is nothing important
    # in the ovn, ovn-nb, or ovs databases.  We're going to trash them and
    # create new ones on each devstack run.

    base_dir=$DATA_DIR/ovs
    mkdir -p $base_dir

    for db in conf.db ovnsb.db ovnnb.db vtep.db ; do
        if [ -f $base_dir/$db ] ; then
            rm -f $base_dir/$db
        fi
    done
    rm -f $base_dir/.*.db.~lock~

    echo "Creating OVS, OVN-Southbound and OVN-Northbound Databases"
    ovsdb-tool create $base_dir/conf.db $DEST/$OVN_REPO_NAME/vswitchd/vswitch.ovsschema
    if is_ovn_service_enabled ovn-northd ; then
        ovsdb-tool create $base_dir/ovnsb.db $DEST/$OVN_REPO_NAME/ovn/ovn-sb.ovsschema
        ovsdb-tool create $base_dir/ovnnb.db $DEST/$OVN_REPO_NAME/ovn/ovn-nb.ovsschema
    fi
    if is_ovn_service_enabled ovn-controller-vtep ; then
        ovsdb-tool create $base_dir/vtep.db $DEST/$OVN_REPO_NAME/vtep/vtep.ovsschema
    fi
}

# install_ovn() - Collect source and prepare
function install_ovn {
    echo "Installing OVN and dependent packages"

    # If OVS is already installed, remove it, because we're about to re-install
    # it from source.
    for package in openvswitch openvswitch-switch openvswitch-common; do
        if is_package_installed $package ; then
            uninstall_package $package
        fi
    done

    if ! is_neutron_enabled ; then
        # networking-ovn depends on neutron, so ensure it at least gets
        # installed.
        install_neutron
    fi

    setup_develop $DEST/networking-ovn
    # Install tox, used to generate the config (see devstack/override-defaults)
    pip_install tox
    source $NEUTRON_DIR/devstack/lib/ovs
    compile_ovs $OVN_BUILD_MODULES
    sudo chown $(whoami) /usr/local/var/run/openvswitch
    sudo chown $(whoami) /usr/local/var/log/openvswitch

    # Install ovs python module from ovs source.
    if [[ "$OVN_INSTALL_OVS_PYTHON_MODULE" == "True" ]]; then
        sudo pip uninstall -y ovs
        sudo pip install -e $DEST/$OVS_REPO_NAME/python
    fi
}

function start_ovs {
    echo "Starting OVS"

    local _pwd=$(pwd)

    local ovsdb_logfile="ovsdb-server.log.${CURRENT_LOG_TIME}"
    bash -c "cd '$LOGDIR' && touch '$ovsdb_logfile' && ln -sf '$ovsdb_logfile' ovsdb-server.log"

    local ovsdb_nb_logfile="ovsdb-server-nb.log.${CURRENT_LOG_TIME}"
    bash -c "cd '$LOGDIR' && touch '$ovsdb_nb_logfile' && ln -sf '$ovsdb_nb_logfile' ovsdb-server-nb.log"

    local ovsdb_sb_logfile="ovsdb-server-sb.log.${CURRENT_LOG_TIME}"
    bash -c "cd '$LOGDIR' && touch '$ovsdb_sb_logfile' && ln -sf '$ovsdb_sb_logfile' ovsdb-server-sb.log"

    cd $DATA_DIR/ovs

    EXTRA_DBS=""
    OVSDB_SB_REMOTE=""
    if is_ovn_service_enabled ovn-northd ; then

        # TODO (regXboi): change ovn-ctl so that we can use something
        # other than --db-nb-port for port and ip address
        DB_NB_PORT="6641"
        DB_NB_FILE="$DATA_DIR/ovs/ovnnb.db"
        OVN_NB_LOGFILE="$LOGDIR/ovsdb-server-nb.log"

        # TODO (regXboi): change ovn-ctl so that we can use something
        # other than --db-sb-port for port and ip address
        DB_SB_PORT="6642"
        DB_SB_FILE="$DATA_DIR/ovs/ovnsb.db"
        OVN_SB_LOGFILE="$LOGDIR/ovsdb-server-sb.log"

        /usr/local/share/openvswitch/scripts/ovn-ctl start_ovsdb \
              --db-nb-port=$DB_NB_PORT --db-sb-port=$DB_SB_PORT \
              --db-nb-file=$DB_NB_FILE --ovn-nb-logfile=$OVN_NB_LOGFILE \
              --db-sb-file=$DB_SB_FILE --ovn-sb-logfile=$OVN_SB_LOGFILE

        echo "Waiting for ovn ovsdb servers to start ... "
        DB_NB_SOCK="/usr/local/var/run/openvswitch/ovnnb_db.sock"
        DB_SB_SOCK="/usr/local/var/run/openvswitch/ovnsb_db.sock"
        local testcmd="test -e $DB_NB_SOCK -a -e $DB_SB_SOCK"
        test_with_retry "$testcmd" "nb ovsdb-server did not start" $SERVICE_TIMEOUT 1
        echo "done."
    fi

    # TODO (regXboi): it would be nice to run the following with run_process
    # and have it end up under the control of screen.  However, at the point
    # this is called, screen isn't running, so we'd have to overload
    # USE_SCREEN to get the process to start, but testing shows that the
    # resulting process doesn't want to create br-int, which leaves things
    # rather broken.  So, stay with this for now and somebody more tenacious
    # than I can figure out how to make it work...

    if is_ovn_service_enabled ovn-controller || is_ovn_service_enabled ovn-controller-vtep ; then
        local _OVSREMOTE="--remote=db:Open_vSwitch,Open_vSwitch,manager_options"
        local _VTEPREMOTE=""
        local _OVSDB=conf.db
        local _VTEPDB=""

        if is_ovn_service_enabled ovn-controller-vtep ; then
            _VTEPREMOTE="--remote=db:hardware_vtep,Global,managers"
            _VTEPDB=vtep.db
        fi

        ovsdb-server --remote=punix:/usr/local/var/run/openvswitch/db.sock \
                     $_OVSREMOTE $_VTEPREMOTE \
                     --pidfile --detach -vconsole:off \
                     --log-file=$LOGDIR/ovsdb-server.log \
                     $_OVSDB $_VTEPDB

        echo -n "Waiting for ovsdb-server to start ... "
        local testcmd="test -e /usr/local/var/run/openvswitch/db.sock"
        test_with_retry "$testcmd" "ovsdb-server did not start" $SERVICE_TIMEOUT 1
        echo "done."
        ovs-vsctl --no-wait init
        ovs-vsctl --no-wait set open_vswitch . system-type="devstack"
        ovs-vsctl --no-wait set open_vswitch . external-ids:system-id="$OVN_UUID"
    fi

    if is_ovn_service_enabled ovn-controller || is_ovn_service_enabled ovn-controller-vtep ; then
        ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-remote="$OVN_SB_REMOTE"
        ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-bridge="br-int"
        ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-type="geneve"
        ovs-vsctl --no-wait set open_vswitch . external-ids:ovn-encap-ip="$HOST_IP"

        ovn_base_setup_bridge br-int
        ovs-vsctl --no-wait set bridge br-int fail-mode=secure other-config:disable-in-band=true

        local ovswd_logfile="ovs-switchd.log.${CURRENT_LOG_TIME}"
        bash -c "cd '$LOGDIR' && touch '$ovswd_logfile' && ln -sf '$ovswd_logfile' ovs-vswitchd.log"

        # Bump up the max number of open files ovs-vswitchd can have
        sudo sh -c "ulimit -n 32000 && exec ovs-vswitchd --pidfile --detach -vconsole:off --log-file=$LOGDIR/ovs-vswitchd.log"

        if is_provider_network || [[ $Q_USE_PROVIDERNET_FOR_PUBLIC == "True" ]]; then
            ovn_base_setup_bridge $OVS_PHYSICAL_BRIDGE
            ovs-vsctl set open . external-ids:ovn-bridge-mappings=${PHYSICAL_NETWORK}:${OVS_PHYSICAL_BRIDGE}
        fi
    fi

    if is_ovn_service_enabled ovn-controller-vtep ; then
        ovn_base_setup_bridge br-vtep
        vtep-ctl add-ps br-vtep
        vtep-ctl set Physical_Switch br-vtep tunnel_ips=$HOST_IP
        sudo /usr/local/share/openvswitch/scripts/ovs-vtep --log-file=$LOGDIR/ovs-vtep.log --pidfile --detach br-vtep
        vtep-ctl set-manager tcp:$HOST_IP:6640
    fi

    cd $_pwd
}

# start_ovn() - Start running processes, including screen
function start_ovn {
    echo "Starting OVN"

    if is_ovn_service_enabled ovn-controller ; then
        # (regXboi) pulling out --log-file to avoid double logging
        # appears to break devstack, so let's not do that
        run_process ovn-controller "sudo ovn-controller --pidfile --log-file unix:/usr/local/var/run/openvswitch/db.sock"

        # This makes sure that the console logs have time stamps to
        # the millisecond, but we need to make sure ovs-appctl has
        # a pid file to work with, so ...
        echo -n "Waiting for ovn-controller to start ... "
        local testcmd="test -e /usr/local/var/run/openvswitch/ovn-controller.pid"
        test_with_retry "$testcmd" "ovn-controller did not start" $SERVICE_TIMEOUT 1
        echo "done."
        sudo ovs-appctl -t ovn-controller vlog/set "PATTERN:CONSOLE:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m"
    fi

    if is_ovn_service_enabled ovn-controller-vtep ; then
        # (regXboi) pulling out --log-file to avoid double logging
        # appears to break devstack, so let's not do that
        run_process ovn-controller-vtep "sudo ovn-controller-vtep --pidfile --log-file --vtep-db=unix:/usr/local/var/run/openvswitch/db.sock --ovnsb-db=$OVN_SB_REMOTE"

        # This makes sure that the console logs have time stamps to
        # the millisecond, but we need to make sure ovs-appctl has
        # a pid file to work with, so ...
        echo -n "Waiting for ovn-controller-vtep to start ... "
        local testcmd="test -e /usr/local/var/run/openvswitch/ovn-controller-vtep.pid"
        test_with_retry "$testcmd" "ovn-controller-vtep did not start" $SERVICE_TIMEOUT 1
        echo "done."
        sudo ovs-appctl -t ovn-controller-vtep vlog/set "PATTERN:CONSOLE:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m"
    fi

    if is_ovn_service_enabled ovn-northd ; then


        run_process ovn-northd "ovn-northd --log-file=$LOGDIR/ovn-northd.log --pidfile"

        # This makes sure that the console logs have time stamps to
        # the millisecond, but we need to make sure ovs-appctl has
        # a pid file to work with, so ...
        echo -n "Waiting for ovn-northd to start ... "
        OVN_NORTHD_PID="/usr/local/var/run/openvswitch/ovn-northd.pid"
        local testcmd="test -e $OVN_NORTHD_PID"
        test_with_retry "$testcmd" "ovn-northd did not start" $SERVICE_TIMEOUT 1
        echo "done."
        sudo ovs-appctl -t ovn-northd vlog/set "PATTERN:CONSOLE:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m"
    fi
}

# stop_ovn() - Stop running processes (non-screen)
function stop_ovn {
    if is_ovn_service_enabled ovn-controller ; then
        stop_process ovn-controller
        sudo killall ovs-vswitchd
    fi
    if is_ovn_service_enabled ovn-controller-vtep ; then
        stop_process ovn-controller-vtep
        sudo killall ovs-vtep
        sudo killall ovs-vswitchd
    fi
    if is_ovn_service_enabled ovn-northd ; then
        /usr/local/share/openvswitch/scripts/ovn-ctl stop_northd
    fi
    sudo killall ovsdb-server
}

# stop_ovs_dp() - Stop OVS datapath
function stop_ovs_dp {
    sudo ovs-dpctl dump-dps | sudo xargs -n1 ovs-dpctl del-dp
    sudo rmmod vport_geneve
    sudo rmmod openvswitch
}

function disable_libvirt_apparmor {
    if ! sudo aa-status --enabled ; then
        return 0
    fi
    # NOTE(arosen): This is used as a work around to allow newer versions
    # of libvirt to work with ovs configured ports. See LP#1466631.
    # requires the apparmor-utils
    install_package apparmor-utils
    # disables apparmor for libvirtd
    sudo aa-complain /etc/apparmor.d/usr.sbin.libvirtd
}

function create_public_bridge {
    # Create the public bridge that OVN will use
    # This logic is based on the devstack neutron-legacy _neutron_configure_router_v4 and _v6
    local ext_gw_ifc
    ext_gw_ifc=$(get_ext_gw_interface)

    sudo ovs-vsctl --may-exist add-br $ext_gw_ifc -- set bridge $ext_gw_ifc protocols=OpenFlow13
    sudo ovs-vsctl set open . external-ids:ovn-bridge-mappings=provider:$ext_gw_ifc
    if [ -n "$FLOATING_RANGE" ]; then
        local cidr_len=${FLOATING_RANGE#*/}
        sudo ip addr add $PUBLIC_NETWORK_GATEWAY/$cidr_len dev $ext_gw_ifc
    fi

    # Ensure IPv6 RAs are accepted on the interface with the default route.
    # This is needed for neutron-based devstack clouds to work in
    # IPv6-only clouds in the gate. Please do not remove this without
    # talking to folks in Infra. This fix is based on a devstack fix for
    # neutron L3 agent: https://review.openstack.org/#/c/359490/.
    default_route_dev=$(ip route | grep ^default | awk '{print $5}')
    sudo sysctl -w net.ipv6.conf.$default_route_dev.accept_ra=2

    sudo sysctl -w net.ipv6.conf.all.forwarding=1
    if [ -n "$IPV6_PUBLIC_RANGE" ]; then
        local ipv6_cidr_len=${IPV6_PUBLIC_RANGE#*/}
        sudo ip -6 addr add $IPV6_PUBLIC_NETWORK_GATEWAY/$ipv6_cidr_len dev $ext_gw_ifc
        sudo ip -6 route replace $FIXED_RANGE_V6 via $IPV6_PUBLIC_NETWORK_GATEWAY dev $ext_gw_ifc
    fi

    sudo ip link set $ext_gw_ifc up
}
