Howto:FreeBSD jail vnet

From Wiki

Jump to: navigation, search
HOWTOS edit

L2TP/IPSEC Linux-iPhoneGentoo LVM LUKSUser level Firewalling (PF, iptables)build a custom DragonFlyBSD-RELEASE kernelVirtual Networking in FreeBSD Jail (bridge + nat)Roadwarrior IPSec on FreeBSDBind9 DLZ with LDAP driver

Contents

Requirements

Image:Warn.png Warning : FreeBSD 8-STABLE and 8.1 doesn't support VIMAGE and PF in kernel (panic on boot)
I search a workarround, if you have any idea you can send me an email to contact_at_davidberard_dot_fr
Image:Warn.png
  • FeeBSD 8.0-RELEASE system
  • FreeBSD sources up to date (use cvsup)
  • world builded

Benefits

  • Full virtualized networking inside jail
  • Jails have there own loopback interface
  • You can use advanced network features inside jail, eg: IPSec.

Build your own custom kernel

Your need to build a custom kernel which will includes VIMAGE and EPAIR(4) devices.

To make jails work on a natted network you'll need PF(4) or IPF(4), and IF_BRIDGE(4) too.

I use the NULLFS option to mount ports tree inside jails.

File: /usr/src/sys/amd64/conf/MYKERNEL

include GENERIC
 
cpu             HAMMER
ident           MYKERNEL
 
# Firewalling
device          pf
device          pflog
 
# Virtual networking for jail
nooptions       SCTP        # 8.0-RELEASE does not support SCTP with VIMAGE
options         VIMAGE
device          epair
device          if_bridge
 
# The nullFS to mount local directory
options         NULLFS


build your kernel :

cd /usr/src
make buildkernel KERNCONF=MYKERNEL
make installkernel KERNCONF=MYKERNEL
# Then reboot your system

Patching your jail rc script

Apply the following patch to /etc/rc.d/jail (see http://www.freebsd.org/cgi/query-pr.cgi?pr=142972)

Note : This patch is for the RELENG_8_0 branch.

File: jail_rc.patch

--- /usr/src/etc/rc.d/jail	2009-10-25 02:10:29.000000000 +0100
+++ /etc/rc.d/jail	2010-05-27 11:44:42.000000000 +0200
@@ -38,6 +38,7 @@
 	_fdescdir="${_devdir}/fd"
 	_procdir="${_rootdir}/proc"
 	eval _hostname=\"\$jail_${_j}_hostname\"
+	eval _name=\"\$jail_${_j}_name\"
 	eval _ip=\"\$jail_${_j}_ip\"
 	eval _interface=\"\${jail_${_j}_interface:-${jail_interface}}\"
 	eval _exec=\"\$jail_${_j}_exec\"
@@ -51,7 +52,7 @@
 
 	eval _exec_start=\"\${jail_${_j}_exec_start:-${jail_exec_start}}\"
 
-	i=1
+	i=0
 	while : ; do
 		eval _exec_afterstart${i}=\"\${jail_${_j}_exec_afterstart${i}:-\${jail_exec_afterstart${i}}}\"
 		[ -z "$(eval echo \"\$_exec_afterstart${i}\")" ] &&  break
@@ -60,6 +61,13 @@
 
 	i=0
 	while : ; do
+		eval _exec_earlypoststart${i}=\"\${jail_${_j}_exec_earlypoststart${i}:-\${jail_exec_earlypoststart${i}}}\"
+		[ -z "$(eval echo \"\$_exec_earlypoststart${i}\")" ] && break
+		i=$((i + 1))
+	done
+
+	i=0
+	while : ; do
 		eval _exec_poststart${i}=\"\${jail_${_j}_exec_poststart${i}:-\${jail_exec_poststart${i}}}\"
 		[ -z "$(eval echo \"\$_exec_poststart${i}\")" ] && break
 		i=$((i + 1))
@@ -94,6 +102,9 @@
 			fi
 		fi
 	fi
+	
+	# JAIL new style
+	eval _v2=\"\${jail_v2_enable:-"NO"}\"
 
 	# The default jail ruleset will be used by rc.subr if none is specified.
 	eval _ruleset=\"\${jail_${_j}_devfs_ruleset:-${jail_devfs_ruleset}}\"
@@ -110,18 +121,26 @@
 	eval _fstab=\"\${jail_${_j}_fstab:-${jail_fstab}}\"
 	[ -z "${_fstab}" ] && _fstab="/etc/fstab.${_j}"
 	eval _flags=\"\${jail_${_j}_flags:-${jail_flags}}\"
-	[ -z "${_flags}" ] && _flags="-l -U root"
+	if checkyesno _v2; then
+		[ -z "${_flags}" ] && _flags="-l -U root -c"
+	else
+		[ -z "${_flags}" ] && _flags="-l -U root"
+	fi
 	eval _consolelog=\"\${jail_${_j}_consolelog:-${jail_consolelog}}\"
 	[ -z "${_consolelog}" ] && _consolelog="/var/log/jail_${_j}_console.log"
 	eval _fib=\"\${jail_${_j}_fib:-${jail_fib}}\"
+	eval _vnet=\"\${jail_${_j}_vnet_enable:-"NO"}\"
 
 	# Debugging aid
 	#
+	debug "$_j v2 enable: $_v2"
 	debug "$_j devfs enable: $_devfs"
 	debug "$_j fdescfs enable: $_fdescfs"
 	debug "$_j procfs enable: $_procfs"
 	debug "$_j mount enable: $_mount"
+	debug "$_j vnet enable: $_vnet"
 	debug "$_j hostname: $_hostname"
+	debug "$_j name: $_name"
 	debug "$_j ip: $_ip"
 	jail_show_addresses ${_j}
 	debug "$_j interface: $_interface"
@@ -145,7 +164,7 @@
 
 	debug "$_j exec start: $_exec_start"
 
-	i=1
+	i=0
 	while : ; do
 		eval out=\"\${_exec_afterstart${i}:-''}\"
 
@@ -480,6 +499,17 @@
 		"")	continue ;;
 		*)	;;
 		esac
+		# Append address to list of addresses for the jail command.
+		case "${_type}" in
+			"inet") case "${_addrlv4}" in
+				"")     _addrlv4="${_addr}" ;;
+				*)      _addrlv4="${_addrlv4},${_addr}" ;;
+				esac;;
+			"inet6") case "${_addrlv6}" in
+				"")     _addrlv6="${_addr}" ;;
+				*)      _addrlv6="${_addrlv6},${_addr}" ;;
+				esac;;
+		esac
 
 		# Append address to list of addresses for the jail command.
 		case "${_addrl}" in
@@ -567,6 +597,8 @@
 			continue;
 		fi
 		_addrl=""
+		_addrlv4=""
+		_addrlv6=""
 		jail_ips "add"
 		if [ -n "${_fib}" ]; then
 			_setfib="setfib -F '${_fib}'"
@@ -635,12 +667,37 @@
 			i=$((i + 1))
 		done
 
-		eval ${_setfib} jail ${_flags} -i ${_rootdir} ${_hostname} \
-			\"${_addrl}\" ${_exec_start} > ${_tmp_jail} 2>&1
+		if checkyesno _v2; then
+			_start_cmd="${_setfib} jail -J ${_tmp_jail} ${_flags} path=${_rootdir} host.hostname=${_hostname} \
+			name=\"${_name}\""
+			if checkyesno _vnet; then
+				_start_cmd="${_start_cmd} vnet"
+			else
+				_start_cmd="${_start_cmd} ip4.addr=\"${_addrlv4}\" ip6.addr=\"${_addrlv6}\""
+			fi
+			_start_cmd="${_start_cmd} command=${_exec_start}"
+			eval ${_start_cmd} > /dev/null 2>&1
+		else
+			eval ${_setfib} jail ${_flags} -i ${_rootdir} ${_hostname} \
+				\"${_addrl}\" ${_exec_start} > ${_tmp_jail} 2>&1
+		fi
 
 		if [ "$?" -eq 0 ] ; then
-			_jail_id=$(head -1 ${_tmp_jail})
-			i=1
+			if checkyesno _v2; then
+				_jail_id=$(awk -F '=| ' '{print $2}' ${_tmp_jail})
+			else
+				_jail_id=$(head -1 ${_tmp_jail})
+			fi
+
+			i=0
+			while : ; do
+				eval out=\"\${_exec_earlypoststart${i}:-''}\"
+				[ -z "$out" ] && break
+				${out}
+				i=$((i + 1))
+			done
+
+			i=0
 			while : ; do
 				eval out=\"\${_exec_afterstart${i}:-''}\"

Prepare the network environement

File: /etc/rc.conf

[--snipped--]
cloned_interfaces="bridge0"
ifconfig_bridge0="inet XX.YY.ZZ.1 netmask 255.255.255.0 up"
 
jail_enable="YES"
jail_v2_enable="YES"
jail_list=""
 
# source jails confs
for file in /etc/jails/*.conf; do
	. $file
done
[--snipped--]
mkdir -p /etc/jails/fstabs
/etc/rc.d/netif cloneup

Create the jail

This example uses ZFS so that every jail is a ZFS dataset. If you choose not to use ZFS, then you'd need to create a directory for the jail's dir.

zpool list
#NAME    SIZE   USED  AVAIL    CAP  HEALTH  ALTROOT
#zpool   344G  15.2G   329G     4%  ONLINE  -
zfs create zpool/example
zfs set mountpoint=/jails/example zpool/example
zfs set quota=10g zpool/example

Now, install FreeBSD userland in the jail dir :

cd /usr/src
make installworld DESTDIR=/jails/example
make distribution DESTDIR=/jails/example
echo 'hostname="example.mydomain.com"' >> /jails/example/etc/rc.conf
mkdir /jails/example/usr/ports

Configure the jail

Note : this is the first jail (epair0, ipaddr can't be duplicate, be careful with multiple jails)

File: /etc/jails/example

#JAIL example
jail_list="$jail_list example"
jail_example_name="example"
jail_example_hostname="example.mydomain.com"
jail_example_devfs_enable="YES"
jail_example_rootdir="/jails/example"
jail_example_mount_enable="YES"
jail_example_fstab="/etc/jails/fstabs/example"
jail_example_vnet_enable="YES"
 
#network
jail_example_exec_prestart0="ifconfig epair0 create"
jail_example_exec_prestart1="ifconfig bridge0 addm epair0a"
jail_example_exec_prestart2="ifconfig epair0a up"
jail_example_exec_earlypoststart0="ifconfig epair0b vnet example"
jail_example_exec_afterstart0="ifconfig lo0 127.0.0.1"
jail_example_exec_afterstart1="ifconfig epair0b XX.YY.ZZ.2 netmask 255.255.255.0 up"
jail_example_exec_afterstart2="route add default XX.YY.ZZ.1"
jail_example_exec_afterstart3="/bin/sh /etc/rc"
jail_example_exec_poststop0="ifconfig bridge0 deletem epair0a"
jail_example_exec_poststop1="ifconfig epair0a destroy"

File: /etc/jails/fstabs/example

/usr/src              /jails/example/usr/src            nullfs      rw     0     0
/usr/ports            /jails/example/usr/ports          nullfs      rw     0     0

Start the jail

/etc/rc.d/jail start example

If all went well :

jls
#   JID  IP Address      Hostname                      Path
#     1  -               example.mydomain.com          /jails/example
 
ifconfig bridge0
#bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
#	ether 3a:43:91:3b:01:e2
#	inet XX.YY.ZZ.1 netmask 0xffffff00 broadcast XX.YY.ZZ.255
#	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
#	maxage 20 holdcnt 6 proto rstp maxaddr 100 timeout 1200
#	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
#	member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
#	        ifmaxaddr 0 port 7 priority 128 path cost 2000
 
ping -q -c 1 XX.YY.ZZ.2
#PING XX.YY.ZZ.2 (XX.YY.ZZ.2): 56 data bytes
#64 bytes from XX.YY.ZZ.2: icmp_seq=0 ttl=64 time=0.078 ms
 
jexec example ifconfig
#lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
#	options=3<RXCSUM,TXCSUM>
#	inet 127.0.0.1 netmask 0xff000000 
#	inet6 ::1 prefixlen 128 
#	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1 
#epair0b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
#	ether 02:00:00:00:07:0b
#	inet6 fe80::ff:fe00:70b%epair1b prefixlen 64 scopeid 0x2 
#	inet XX.YY.ZZ.2 netmask 0xffffff00 broadcast XX.YY.ZZ.255

Advenced networking : NAT and firewalling

  • Enable IPv4 and/or IPv6 forwarding
echo 'gateway_enable="YES"' >> /etc/rc.conf
echo 'ipv6_gateway_enable="YES"' >> /etc/rc.conf
sysctl -w net.inet.ip.forwarding=1
sysctl -w net.inet6.ip6.forwarding=1
  • Enable OpenBSD Packet filter
echo '
# PF
pf_enable="YES"
pf_rules="/etc/pf.conf"
pflog_enable="YES"
' >> /etc/rc.conf
  • Configure network address translation

File: /etc/pf.conf

pub="X.X.X.X" # my public address
jail_net="XX.YY.ZZ.0/24"
example_jail="XX.YY.ZZ.2"
if="em0"
 
set block-policy return
set skip on lo
scrub in
 
# NAT
nat on $if from $example_jail to !$jail_net -> $pub
# redirect port 222 inside jail
rdr on $if proto tcp from any to $pub port 222 -> $example_jail port ssh
 
# default
pass out on $if from $pub to any
block in log on $if
 
# example jail ssh
pass in quick on $if proto tcp from any to $example_jail port ssh
 
# ssh on the host machine
pass in quick on $if proto tcp from any to $pub port ssh
# load firewall rules
pfctl -ef /etc/pf.conf
  • Don't forget DNS in jail
echo "nameserver Y.Y.Y.Y" >> /jails/example/etc/resolv.conf
  • Try it out
jexec example ping -c 1 google.com
#PING google.com (209.85.229.99): 56 data bytes
#64 bytes from 209.85.229.99: icmp_seq=0 ttl=52 time=31.866 ms
#--- google.com ping statistics ---
#1 packets transmitted, 1 packets received, 0.0% packet loss
#round-trip min/avg/max/stddev = 31.866/31.866/31.866/0.000 m
  • Enter in the jail as root
jexec example login -f root
Personal tools