A short overview about how to install an ESXi5 server via PXE boot using the following steps.

1. Set up a PXE boot server
2. Copy the content of the ESXi5 ISO to a directory
3. Edit VMware's boot.cfg file
4. Create a ks.cfg file for unattended installation
5. Edit the PXE boot config file

Step 1 Set up a PXE boot server

Step 2 Copy the content of the ESXi5 ISO to a directory

mkdir /mnt/cdrom
mkdir /tftpboot/images/vsphere/esxi5.0
mount -o loop VMware-VMvisor-Installer-5.0.0-469512.x86_64.iso /media/tmp
or
mount /dev/cdrom/ /mnt/cdrom
cp -rav /mnt/cdrom/* /tftpboot/images/vsphere/esxi5.0
chmod -R +w /tftpboot/images/vsphere/esxi5.0

Step 3: Edit VMware's boot.cfg file

vi /tftpboot/images/vsphere/esxi5.0/boot.cfg

bootstate=0
title=Loading ESXi installer
prefix=/images/vmware/esxi/5.0/
kernel=tboot.b00
#kernelopt=runweasel
modules=b.b00 --- useropts.gz --- k.b00 --- a.b00 --- ata-pata.v00 --- ata-pata.v01 --- ata-pata.v02 --- ata-pata.v03 --- ata-pata.v04 
--- ata-pata.v05 --- ata-pata.v06 --- ata-pata.v07 --- block-cc.v00 --- ehci-ehc.v00 --- s.v00 --- weaselin.i00 --- ima-qla4.v00 --- ipmi-ipm.v00 
--- ipmi-ipm.v01 --- ipmi-ipm.v02 --- misc-cni.v00 --- misc-dri.v00 --- net-be2n.v00 --- net-bnx2.v00 --- net-bnx2.v01 --- net-cnic.v00 
--- net-e100.v00 --- net-e100.v01 --- net-enic.v00 --- net-forc.v00 --- net-igb.v00 --- net-ixgb.v00 --- net-nx-n.v00 --- net-r816.v00 
--- net-r816.v01 --- net-s2io.v00 --- net-sky2.v00 --- net-tg3.v00 --- ohci-usb.v00 --- sata-ahc.v00 --- sata-ata.v00 --- sata-sat.v00 
--- sata-sat.v01 --- sata-sat.v02 --- sata-sat.v03 --- scsi-aac.v00 --- scsi-adp.v00 --- scsi-aic.v00 --- scsi-bnx.v00 --- scsi-fni.v00 
--- scsi-hps.v00 --- scsi-ips.v00 --- scsi-lpf.v00 --- scsi-meg.v00 --- scsi-meg.v01 --- scsi-meg.v02 --- scsi-mpt.v00 --- scsi-mpt.v01 
--- scsi-mpt.v02 --- scsi-qla.v00 --- scsi-qla.v01 --- uhci-usb.v00 --- tools.t00 --- imgdb.tgz --- imgpayld.tgz
build=
updated=0

bootstate=0
title=Loading ESXi installer
kernel=http://10.0.2.14:8080/vSphere/ESXi_5.0/tboot.b00
kernelopt=runweasel
#kernelopt=ks=http://10.0.2.14:8080/esxi_ksFiles/ks.cfg
modules=http://10.0.2.14:8080/vSphere/ESXi_5.0/b.b00 --- http://10.0.2.14:8080/vSphere/ESXi_5.0/useropts.gz --- http://10.0.2.14:8080/vSphere/ESXi_5.0/k.b00 
--- http://10.0.2.14:8080/vSphere/ESXi_5.0/a.b00 --- http://10.0.2.14:8080/vSphere/ESXi_5.0/s.v00 --- http://10.0.2.14:8080/vSphere/ESXi_5.0/weaselin.v00 
--- http://10.0.2.14:8080/vSphere/ESXi_5.0/tools.t00 --- http://10.0.2.14:8080/vSphere/ESXi_5.0/imgdb.tgz --- http://10.0.2.14:8080/vSphere/ESXi_5.0/imgpayld.tgz

Step 4: Create a ks.cfg script

accepteula
#dryrun
install --firstdisk --overwritevmfs
#url --url http://10.0.2.11/vmware/esxi/5.0/ks.cfg
rootpw password
reboot
network --bootproto=static --ip=10.0.2.100 --gateway=10.0.1.254 --nameserver=10.0.2.11 --netmask=255.0.0.0 --hostname=esxi5.r71.nl --addvmportgroup=1

%firstboot --interpreter=busybox

# enable HV (Hardware Virtualization to run nested 64bit Guests + Hyper-V VM)
#grep -i "vhv.allow" /etc/vmware/config || echo "vhv.allow = \"TRUE\"" >> /etc/vmware/config
 
# enable & start remote ESXi Shell  (SSH)
vim-cmd hostsvc/enable_ssh
vim-cmd hostsvc/start_ssh
 
# enable & start ESXi Shell (TSM)
vim-cmd hostsvc/enable_esx_shell
vim-cmd hostsvc/start_esx_shell

Make the script available on a webserver. Use the URL to the file in step 5.

Step 5: Edit the PXE boot config file

vi /tftpboot/pxelinux.cfg/default

LABEL ESXi5
KERNEL images/vsphere/esxi5/mboot.c32
APPEND -c images/vsphere/esxi5/boot.cfg pxebooting ks=http://10.0.2.14:8080/vSphere/ESXi_5.0/ks.cfg +++
IPAPPEND 1