Linux Boot Server
Setting of the Linux server to be used for PXE and EFI boot of the VME controllers
Login to the server as root. Copy two files to the root directory and untar them:
cd / cp /usr/downloads/tftpboot.tar.gz_clondaq15 tftpboot.tar.gz cp /usr/downloads/diskless.tar.gz_clondaq15 diskless.tar.gz gunzip tftpboot.tar.gz gunzip diskless.tar.gz tar xvf tftpboot.tar tar xvf diskless.tar rm tftpboot.tar diskless.tar
How to add new client to the system
Add new record to:
/var/named/10.168.192.in-addr.arpa.db /var/named/clontest.com /etc/dhcp/dhcpd.conf
Restart corresponding services:
systemctl restart dhcpd systemctl restart named
Check services status:
systemctl status dhcpd systemctl status named
IPTABLES
Do not forget:
chattr +i /etc/resolv.conf
Show tables:
iptables -vL -t filter iptables -vL -t nat iptables -vL -t mangle iptables -vL -t raw iptables -vL -t security
Only first two seems relevant. Clear them:
iptables -t filter -F iptables -t nat -F
Set needed settings (we assumes loval network port is 'enp_bond', and uplink port is 'em1'):
iptables -A INPUT -i lo -j ACCEPT iptables -A INPUT -i enp_bond -j ACCEPT iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT iptables -t nat -A POSTROUTING -o em1 -j MASQUERADE iptables -A FORWARD -i em1 -o enp_bond -m state --state RELATED,ESTABLISHED -j ACCEPT iptables -A FORWARD -i enp_bond -o em1 -j ACCEPT
Save rules and restart service:
iptables-save > /etc/sysconfig/iptables systemctl enable iptables systemctl start iptables
MODIFY /diskless/CentOS7/x86_64/root/etc/sysconfig/readonly-root:
... CLIENTSTATE=192.168.10.1:/diskless/CentOS7/x86_64/snapshot ...
In root area, /etc/ssh/sshd_config must have following setting (to allow remote ssh)
#UsePAM yes
it will be propagated to the controller's snapshot area on the first boot.
Not sure about others:
HostbasedAuthentication yes #HostbasedAuthentication no
IgnoreRhosts no #IgnoreRhosts yes
PasswordAuthentication yes #PasswordAuthentication yes PasswordAuthentication yes
#UsePrivilegeSeparation sandbox # Default for new installations. UsePrivilegeSeparation sandbox # Default for new installations.
old gefvme removal
yum remove kmod-gefvme gefvme-library
If failed on 'gefvme-library', do
rpm -e --noscripts gefvme-library
If file /etc/modules-load.d/gefvme.conf was not removed by above commands, remove it manually (if working from chroot ... then everything is done in root area, but gefvme.conf may survive in snapshot area, in that case remove it on controller and not in chroot).
File /etc/modules-load.d/vme.conf with contents
cmem_rcc jvme vme_vivo vme_ca91cx42 vme_tsi148 vme
may not be needed (if modules already in kernel ?).
new jvme (not sure if module installation is needed, maybe in kernel already ?)
On VME controller in BIOS, make sure VME memry size is set to 512M
Project git link: https://code.jlab.org/fedaq/drivers/jvme/-/tree/release-3.0
On vme controller as boiarino, copy jvme-release-3.0.tar to $CODA/src and untar it.
Create two environment scripts:
jvme_bash:
#!/bin/bash
export LINUXVME=${CODA}/src/jvme-release-3.0/linuxvme
export LINUXVME_INC=${LINUXVME}/include
export LINUXVME_LIB=${LINUXVME}/Linux_`uname -m`_vme/lib
export LINUXVME_BIN=${LINUXVME}/Linux_`uname -m`_vme/bin
export LD_LIBRARY_PATH=${LINUXVME_LIB}:${LD_LIBRARY_PATH}
export KERNELRELEASE=3.10.0-1062.9.1.el7.x86_64
jvme_tcsh:
#!/bin/tcsh
setenv LINUXVME ${CODA}/src/jvme-release-3.0/linuxvme
setenv LINUXVME_INC ${LINUXVME}/include
setenv LINUXVME_LIB ${LINUXVME}/Linux_`uname -m`_vme/lib
setenv LINUXVME_BIN ${LINUXVME}/Linux_`uname -m`_vme/bin
setenv LD_LIBRARY_PATH ${LINUXVME_LIB}:${LD_LIBRARY_PATH}
setenv KERNELRELEASE 3.10.0-1062.9.1.el7.x86_64
Run source jvme_tcsh. Go inside jvme-release-3.0. In two files CMakeLists.txt and src/CMakeLists.txt, change
set(libpath Linux-${CMAKE_SYSTEM_PROCESSOR}/lib)
set(libpath Linux-${CMAKE_SYSTEM_PROCESSOR}/bin)
to
set(libpath Linux_${CMAKE_SYSTEM_PROCESSOR}_vme/lib)
set(libpath Linux_${CMAKE_SYSTEM_PROCESSOR}_vme/bin)
Type cmake -B build -S . -DCMAKE_INSTALL_PREFIX=$LINUXVME
Fix Makefile in kernel_driver and three it's subdirectories, it must have following in the beginning:
KVERSION := $(KERNELRELEASE) ifeq ($(origin KERNELRELEASE), undefined) KVERSION := $(shell uname -r) endif
In directory jvme-release-3.0, type make and make install,
Do cd kernel_driver and make (do NOT do make install).
On the server, do
mount -o bind /usr/clas12 /diskless/CentOS7/x86_64/root/usr/clas12 mount -o bind /usr/local /diskless/CentOS7/x86_64/root/usr/local mount -o bind /home /diskless/CentOS7/x86_64/root/home chroot /diskless/CentOS7/x86_64/root cd /usr/clas12/release/2.0.0/coda/src source jvme_bash cd jvme-release-3.0/kernel_driver make install
Still on server, add two files to /etc/udev/rules.d directory:
99-cmem.rules:
KERNEL=="cmem_rcc", MODE="0666"
99-vme.rules:
KERNEL=="bus/vme/ctl", MODE="0666" KERNEL=="bus/vme/m_a16", MODE="0666" KERNEL=="bus/vme/m_a24", MODE="0666" KERNEL=="bus/vme/m_a32", MODE="0666" KERNEL=="bus/vme/m_crcsr", MODE="0666" KERNEL=="bus/vme/s_a32", MODE="0666" KERNEL=="bus/vme/s_rsvd1", MODE="0666" KERNEL=="bus/vme/s_rsvd2", MODE="0666" KERNEL=="bus/vme/s_rsvd3", MODE="0666"
Reboot controller, check if everything is good.
NOTE: if changing something in kernel module(s), it is not needed to reboot every time after make/make install is done in jvme-release-3.0/kernel_driver directory. Just run ./load_driver.sh as root on controller, and all modules will be reloaded.
NOTE: to add /et to snapshot area: on server, create /et directories in both root and snapshot areas, then add line '/et' to /etc/statetab file in root area, then reboot controller.
Enable systemd log persistency (remember all reboots, not only last one)
Run emacs /etc/systemd/journald.conf, set
Storage=persistent
Do following:
mkdir /var/log/journal systemd-tmpfiles --create --prefix /var/log/journal systemctl restart systemd-journald
yum
On server where vme is loading from (as root; /zzz will be needed for mysqltcl installation below):
mkdir /diskless/CentOS7/x86_64/root/zzz #mount -o bind /usr/local/src /diskless/CentOS7/x86_64/root/zzz mount -o bind /usr/local /diskless/CentOS7/x86_64/root/usr/local mount -o bind /usr/clas12 /diskless/CentOS7/x86_64/root/usr/clas12 chroot /diskless/CentOS7/x86_64/root
Add multilib_policy=all to /etc/yum.conf.
Add (and remove the rest ?) following to /etc/yum.repos.d/CentOS-Base.repo:
[base] name=CentOS-$releasever - Base baseurl=http://archive.kernel.org/centos-vault/centos/$releasever/os/$basearch/
Clean yum database:
rm /var/lib/rpm/__db.*
Install following using yum:
yum install motif-devel tcl-devel tk-devel libXpm-devel apr-devel libXaw-devel ncurses-devel
Install following for dbedit:
yum install tix itcl itk
Install remaining tcl stuff from /usr/local/src, mounted as /zzz above:
cd /zzz/mysqltcl-3.052 make install ln -s /usr/lib/mysqltcl-3.052 /usr/lib64/tcl8.5/mysqltcl-3.052
To enable remove ssh login, in 'sshd_config comment out this: #UsePAM yes
Gateway
Add
net.ipv4.ip_forward=1
to /etc/sysctl.conf and execute
sysctl -p
NIS server
yum install ypserv rpcbind
systemctl start ypserv
/etc/hosts:
192.168.10.1 clondaq15daq1.clontest.com clondaq15daq1 192.168.10.5 test5.clontest.com test5 192.168.10.6 test6.clontest.com test6
On clondaq15:
route add -net 192.168.10.0 netmask 255.255.255.0 gw 129.57.86.1
route:
Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface default gw-86.jlab.org 0.0.0.0 UG 101 0 0 em1 129.57.86.0 0.0.0.0 255.255.255.0 U 101 0 0 em1 192.168.10.0 gw-86.jlab.org 255.255.255.0 UG 0 0 0 em1 192.168.10.0 0.0.0.0 255.255.255.0 U 102 0 0 p2p1
Setting DHCP server on clon10new (RHEL7)
yum install dhcp tftp tftp-server
Bryan:
yum install nfs-utils tftp-server syslinux-tftpboot syslinux
yum install dnsmasq
mkdir /tftpboot
grub2-mknetdir --net-directory=/tftpboot/efi
Create file /etc/systemd/system/tftp.service:
[Unit] Description=Tftp Server Requires=tftp.socket Documentation=man:in.tftpd
[Service] #ExecStart=/usr/sbin/in.tftpd -s /var/lib/tftpboot ExecStart=/usr/sbin/in.tftpd -s /tftpboot StandardInput=socket
[Install] Also=tftp.socket
Start tftp:
systemctl start tftp #systemctl start tftp.socket ??
Create file /tftpboot/efi/boot/grub2/grub.cfg with following contents:
function load_video {
insmod efi_gop
insmod efi_uga
insmod video_bochs
insmod video_cirrus
insmod all_video
}
load_video
set gfxpayload=keep
insmod gzio
set timeout=2
menuentry 'Diskless CentOS7 x86_64, any network device' --class redhat --class gnu-linux --class gnu --class os {
linuxefi linux-install/CentOS7-x86_64-Diskless/vmlinuz-3.10.0-1062.9.1.el7.x86_64 zram=1 ip=::::::dhcp root=nfs:192.168.10.1:/diskless/CentOS7-devel/x86_64/root ro vga=0x305 module_blacklist=ipmi_si,ipmi_msghandler,ipmi_devintf,w83977f_wdt
initrdefi linux-install/CentOS7-x86_64-Diskless/initramfs-jvme-3.10.0-1062.9.1.el7.x86_64.img
}
Edit file /etc/dhcp/dhcpd.conf:
subnet 192.168.10.0 netmask 255.255.255.0 {
option domain-name "jlab.org";
option domain-name-servers 129.57.32.100, 129.57.32.101;
option routers 192.168.10.1;
use-host-decl-names true;
pool {
range 192.168.10.2 192.168.10.20;
deny dynamic bootp clients;
allow unknown clients;
}
}
set vendorclass = option vendor-class-identifier; option pxe-system-type code 93 = unsigned integer 16; set pxetype = option pxe-system-type;
# DISKLESS Clients in here
group
{
if substring(vendorclass, 0, 9)="PXEClient" {
if pxetype=00:06 or pxetype=00:07 {
filename "efi/boot/grub2/x86_64-efi/core.efi";
} else {
filename "linux-install/pxelinux.0";
}
}
next-server 192.168.10.1;
host test1 {
hardware ethernet 00:20:38:03:10:34;
fixed-address 192.168.10.4;
}
host test4 {
hardware ethernet 00:20:38:10:14:f7;
fixed-address 192.168.10.5;
}
} # Diskless clients group
Start dhcp:
systemctl start dhcpd
Install nfs:
yum install nfs-utils
Configure file /etc/exports:
/diskless 192.168.10.0/24(rw,no_root_squash,sync)
Start NFS server:
systemctl status nfs-server
Check that NFS is exporting. Command
showmount -e
have to show following:
Export list for clondaq15.jlab.org: /diskless 192.168.10.0/24
To use local name server, install bind:
yum install bind
Login from console may not work because of file /etc/securetty permissions, it must be 644.
THere is a service PAM, it may prevent login from console if some required services did not start. To work around, comment out some lines in /etc/pam.d/system-auth-ac file:
#%PAM-1.0 # This file is auto-generated. # User changes will be destroyed the next time authconfig is run. #auth required pam_env.so auth sufficient pam_unix.so nullok try_first_pass #auth requisite pam_succeed_if.so uid >= 1000 quiet_success #auth required pam_deny.so #account required pam_unix.so account sufficient pam_localuser.so account sufficient pam_succeed_if.so uid < 1000 quiet #account required pam_permit.so password requisite pam_pwquality.so try_first_pass local_users_only retry=3 authtok_type= password sufficient pam_unix.so sha512 shadow nullok try_first_pass use_authtok #password required pam_deny.so session optional pam_keyinit.so revoke #session required pam_limits.so -session optional pam_systemd.so session [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid #session required pam_unix.so