Howto:FreeBSD jail vnet
From Wiki
| ||
|---|---|---|
|
L2TP/IPSEC Linux-iPhone • Gentoo LVM LUKS • User level Firewalling (PF, iptables) • build a custom DragonFlyBSD-RELEASE kernel • Virtual Networking in FreeBSD Jail (bridge + nat) • Roadwarrior IPSec on FreeBSD • Bind9 DLZ with LDAP driver |
Contents |
Requirements
| Warning : FreeBSD 8-STABLE and 8.1 doesn't support VIMAGE and PF in kernel (panic on boot) I search a workarround, if you have any idea you can send me an email to contact_at_davidberard_dot_fr |
- FeeBSD 8.0-RELEASE system
- FreeBSD sources up to date (use cvsup)
- world builded
Benefits
- Full virtualized networking inside jail
- Jails have there own loopback interface
- You can use advanced network features inside jail, eg: IPSec.
Build your own custom kernel
Your need to build a custom kernel which will includes VIMAGE and EPAIR(4) devices.
To make jails work on a natted network you'll need PF(4) or IPF(4), and IF_BRIDGE(4) too.
I use the NULLFS option to mount ports tree inside jails.
File: /usr/src/sys/amd64/conf/MYKERNEL
include GENERIC cpu HAMMER ident MYKERNEL # Firewalling device pf device pflog # Virtual networking for jail nooptions SCTP # 8.0-RELEASE does not support SCTP with VIMAGE options VIMAGE device epair device if_bridge # The nullFS to mount local directory options NULLFS
build your kernel :
cd /usr/src make buildkernel KERNCONF=MYKERNEL make installkernel KERNCONF=MYKERNEL # Then reboot your system
Patching your jail rc script
Apply the following patch to /etc/rc.d/jail (see http://www.freebsd.org/cgi/query-pr.cgi?pr=142972)
Note : This patch is for the RELENG_8_0 branch.
File: jail_rc.patch
--- /usr/src/etc/rc.d/jail 2009-10-25 02:10:29.000000000 +0100 +++ /etc/rc.d/jail 2010-05-27 11:44:42.000000000 +0200 @@ -38,6 +38,7 @@ _fdescdir="${_devdir}/fd" _procdir="${_rootdir}/proc" eval _hostname=\"\$jail_${_j}_hostname\" + eval _name=\"\$jail_${_j}_name\" eval _ip=\"\$jail_${_j}_ip\" eval _interface=\"\${jail_${_j}_interface:-${jail_interface}}\" eval _exec=\"\$jail_${_j}_exec\" @@ -51,7 +52,7 @@ eval _exec_start=\"\${jail_${_j}_exec_start:-${jail_exec_start}}\" - i=1 + i=0 while : ; do eval _exec_afterstart${i}=\"\${jail_${_j}_exec_afterstart${i}:-\${jail_exec_afterstart${i}}}\" [ -z "$(eval echo \"\$_exec_afterstart${i}\")" ] && break @@ -60,6 +61,13 @@ i=0 while : ; do + eval _exec_earlypoststart${i}=\"\${jail_${_j}_exec_earlypoststart${i}:-\${jail_exec_earlypoststart${i}}}\" + [ -z "$(eval echo \"\$_exec_earlypoststart${i}\")" ] && break + i=$((i + 1)) + done + + i=0 + while : ; do eval _exec_poststart${i}=\"\${jail_${_j}_exec_poststart${i}:-\${jail_exec_poststart${i}}}\" [ -z "$(eval echo \"\$_exec_poststart${i}\")" ] && break i=$((i + 1)) @@ -94,6 +102,9 @@ fi fi fi + + # JAIL new style + eval _v2=\"\${jail_v2_enable:-"NO"}\" # The default jail ruleset will be used by rc.subr if none is specified. eval _ruleset=\"\${jail_${_j}_devfs_ruleset:-${jail_devfs_ruleset}}\" @@ -110,18 +121,26 @@ eval _fstab=\"\${jail_${_j}_fstab:-${jail_fstab}}\" [ -z "${_fstab}" ] && _fstab="/etc/fstab.${_j}" eval _flags=\"\${jail_${_j}_flags:-${jail_flags}}\" - [ -z "${_flags}" ] && _flags="-l -U root" + if checkyesno _v2; then + [ -z "${_flags}" ] && _flags="-l -U root -c" + else + [ -z "${_flags}" ] && _flags="-l -U root" + fi eval _consolelog=\"\${jail_${_j}_consolelog:-${jail_consolelog}}\" [ -z "${_consolelog}" ] && _consolelog="/var/log/jail_${_j}_console.log" eval _fib=\"\${jail_${_j}_fib:-${jail_fib}}\" + eval _vnet=\"\${jail_${_j}_vnet_enable:-"NO"}\" # Debugging aid # + debug "$_j v2 enable: $_v2" debug "$_j devfs enable: $_devfs" debug "$_j fdescfs enable: $_fdescfs" debug "$_j procfs enable: $_procfs" debug "$_j mount enable: $_mount" + debug "$_j vnet enable: $_vnet" debug "$_j hostname: $_hostname" + debug "$_j name: $_name" debug "$_j ip: $_ip" jail_show_addresses ${_j} debug "$_j interface: $_interface" @@ -145,7 +164,7 @@ debug "$_j exec start: $_exec_start" - i=1 + i=0 while : ; do eval out=\"\${_exec_afterstart${i}:-''}\" @@ -480,6 +499,17 @@ "") continue ;; *) ;; esac + # Append address to list of addresses for the jail command. + case "${_type}" in + "inet") case "${_addrlv4}" in + "") _addrlv4="${_addr}" ;; + *) _addrlv4="${_addrlv4},${_addr}" ;; + esac;; + "inet6") case "${_addrlv6}" in + "") _addrlv6="${_addr}" ;; + *) _addrlv6="${_addrlv6},${_addr}" ;; + esac;; + esac # Append address to list of addresses for the jail command. case "${_addrl}" in @@ -567,6 +597,8 @@ continue; fi _addrl="" + _addrlv4="" + _addrlv6="" jail_ips "add" if [ -n "${_fib}" ]; then _setfib="setfib -F '${_fib}'" @@ -635,12 +667,37 @@ i=$((i + 1)) done - eval ${_setfib} jail ${_flags} -i ${_rootdir} ${_hostname} \ - \"${_addrl}\" ${_exec_start} > ${_tmp_jail} 2>&1 + if checkyesno _v2; then + _start_cmd="${_setfib} jail -J ${_tmp_jail} ${_flags} path=${_rootdir} host.hostname=${_hostname} \ + name=\"${_name}\"" + if checkyesno _vnet; then + _start_cmd="${_start_cmd} vnet" + else + _start_cmd="${_start_cmd} ip4.addr=\"${_addrlv4}\" ip6.addr=\"${_addrlv6}\"" + fi + _start_cmd="${_start_cmd} command=${_exec_start}" + eval ${_start_cmd} > /dev/null 2>&1 + else + eval ${_setfib} jail ${_flags} -i ${_rootdir} ${_hostname} \ + \"${_addrl}\" ${_exec_start} > ${_tmp_jail} 2>&1 + fi if [ "$?" -eq 0 ] ; then - _jail_id=$(head -1 ${_tmp_jail}) - i=1 + if checkyesno _v2; then + _jail_id=$(awk -F '=| ' '{print $2}' ${_tmp_jail}) + else + _jail_id=$(head -1 ${_tmp_jail}) + fi + + i=0 + while : ; do + eval out=\"\${_exec_earlypoststart${i}:-''}\" + [ -z "$out" ] && break + ${out} + i=$((i + 1)) + done + + i=0 while : ; do eval out=\"\${_exec_afterstart${i}:-''}\"
Prepare the network environement
File: /etc/rc.conf
[--snipped--] cloned_interfaces="bridge0" ifconfig_bridge0="inet XX.YY.ZZ.1 netmask 255.255.255.0 up" jail_enable="YES" jail_v2_enable="YES" jail_list="" # source jails confs for file in /etc/jails/*.conf; do . $file done [--snipped--]
mkdir -p /etc/jails/fstabs
/etc/rc.d/netif cloneup
Create the jail
This example uses ZFS so that every jail is a ZFS dataset. If you choose not to use ZFS, then you'd need to create a directory for the jail's dir.
zpool list #NAME SIZE USED AVAIL CAP HEALTH ALTROOT #zpool 344G 15.2G 329G 4% ONLINE - zfs create zpool/example zfs set mountpoint=/jails/example zpool/example zfs set quota=10g zpool/example
Now, install FreeBSD userland in the jail dir :
cd /usr/src make installworld DESTDIR=/jails/example make distribution DESTDIR=/jails/example echo 'hostname="example.mydomain.com"' >> /jails/example/etc/rc.conf mkdir /jails/example/usr/ports
Configure the jail
Note : this is the first jail (epair0, ipaddr can't be duplicate, be careful with multiple jails)
File: /etc/jails/example
#JAIL example jail_list="$jail_list example" jail_example_name="example" jail_example_hostname="example.mydomain.com" jail_example_devfs_enable="YES" jail_example_rootdir="/jails/example" jail_example_mount_enable="YES" jail_example_fstab="/etc/jails/fstabs/example" jail_example_vnet_enable="YES" #network jail_example_exec_prestart0="ifconfig epair0 create" jail_example_exec_prestart1="ifconfig bridge0 addm epair0a" jail_example_exec_prestart2="ifconfig epair0a up" jail_example_exec_earlypoststart0="ifconfig epair0b vnet example" jail_example_exec_afterstart0="ifconfig lo0 127.0.0.1" jail_example_exec_afterstart1="ifconfig epair0b XX.YY.ZZ.2 netmask 255.255.255.0 up" jail_example_exec_afterstart2="route add default XX.YY.ZZ.1" jail_example_exec_afterstart3="/bin/sh /etc/rc" jail_example_exec_poststop0="ifconfig bridge0 deletem epair0a" jail_example_exec_poststop1="ifconfig epair0a destroy"
File: /etc/jails/fstabs/example
/usr/src /jails/example/usr/src nullfs rw 0 0 /usr/ports /jails/example/usr/ports nullfs rw 0 0
Start the jail
/etc/rc.d/jail start example
If all went well :
jls # JID IP Address Hostname Path # 1 - example.mydomain.com /jails/example ifconfig bridge0 #bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 # ether 3a:43:91:3b:01:e2 # inet XX.YY.ZZ.1 netmask 0xffffff00 broadcast XX.YY.ZZ.255 # id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15 # maxage 20 holdcnt 6 proto rstp maxaddr 100 timeout 1200 # root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0 # member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP> # ifmaxaddr 0 port 7 priority 128 path cost 2000 ping -q -c 1 XX.YY.ZZ.2 #PING XX.YY.ZZ.2 (XX.YY.ZZ.2): 56 data bytes #64 bytes from XX.YY.ZZ.2: icmp_seq=0 ttl=64 time=0.078 ms jexec example ifconfig #lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384 # options=3<RXCSUM,TXCSUM> # inet 127.0.0.1 netmask 0xff000000 # inet6 ::1 prefixlen 128 # inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1 #epair0b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500 # ether 02:00:00:00:07:0b # inet6 fe80::ff:fe00:70b%epair1b prefixlen 64 scopeid 0x2 # inet XX.YY.ZZ.2 netmask 0xffffff00 broadcast XX.YY.ZZ.255
Advenced networking : NAT and firewalling
- Enable IPv4 and/or IPv6 forwarding
echo 'gateway_enable="YES"' >> /etc/rc.conf echo 'ipv6_gateway_enable="YES"' >> /etc/rc.conf sysctl -w net.inet.ip.forwarding=1 sysctl -w net.inet6.ip6.forwarding=1
- Enable OpenBSD Packet filter
echo ' # PF pf_enable="YES" pf_rules="/etc/pf.conf" pflog_enable="YES" ' >> /etc/rc.conf
- Configure network address translation
File: /etc/pf.conf
pub="X.X.X.X" # my public address jail_net="XX.YY.ZZ.0/24" example_jail="XX.YY.ZZ.2" if="em0" set block-policy return set skip on lo scrub in # NAT nat on $if from $example_jail to !$jail_net -> $pub # redirect port 222 inside jail rdr on $if proto tcp from any to $pub port 222 -> $example_jail port ssh # default pass out on $if from $pub to any block in log on $if # example jail ssh pass in quick on $if proto tcp from any to $example_jail port ssh # ssh on the host machine pass in quick on $if proto tcp from any to $pub port ssh
# load firewall rules pfctl -ef /etc/pf.conf
- Don't forget DNS in jail
echo "nameserver Y.Y.Y.Y" >> /jails/example/etc/resolv.conf
- Try it out
jexec example ping -c 1 google.com #PING google.com (209.85.229.99): 56 data bytes #64 bytes from 209.85.229.99: icmp_seq=0 ttl=52 time=31.866 ms #--- google.com ping statistics --- #1 packets transmitted, 1 packets received, 0.0% packet loss #round-trip min/avg/max/stddev = 31.866/31.866/31.866/0.000 m
- Enter in the jail as root
jexec example login -f root

