I want to combine NAS storage functionality with Openstack KVM instances on one box. Cinder can use LVM as storage backend and I want to use bcache as SSD cacing in front of mdraid. On top runs LVM which is necessary for Cinder.

Why Debian? Debian 9 has got a very new kernel which is very wishful for using bcache. The bcache utilities are default in Debian repos. Ubuntu 16.04 LTS is not as new as the long lasting Debian 9.

apt install chrony

/etc/chrony/chrony.conf

.
.
server 172.16.2.250 iburst
.
.

systemctl restart chrony
systemctl enable chrony

apt install ifenslave-2.6 vlan

/etc/network/interfaces

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

source /etc/network/interfaces.d/*

# The loopback network interface
auto lo
iface lo inet loopback

auto bond0
iface bond0 inet manual
  bond_mode 802.3ad
  bond_miimon 100
  bond_xmit_hash_policy layer2+3
  bond_updelay 200
  bond_downdelay 200
  bond_lacp_rate fast
  slaves eno1 eno2

auto vlan2
iface vlan2 inet static
  vlan_raw_device bond0
  address 172.16.2.3/24
  gateway 172.16.2.1

iface vlan2 inet6 static
  address 2a02:22a0:bbb7:402::3/64
  gateway 2a02:22a0:bbb7:402::1

auto vlan11
iface vlan11 inet static
  vlan_raw_device bond0
  address 172.16.3.3/24

iface vlan11 inet6 static
   address 2a02:22a0:bbb7:403::3/64

# dns-* options are implemented by the resolvconf package, if installed
dns-nameservers 2a02:22a0:bbb7:402::21 172.16.2.21 2620:0:ccc::2
dns-search mngt.bh.helux.nl

apt install sssd sssd-ldap

/etc/sssd/sssd.conf

[sssd]
config_file_version = 2
services = nss, pam
domains = LDAP

[nss]

[pam]

[domain/LDAP]
id_provider = ldap
auth_provider = ldap

ldap_uri = ldap://ldap.mngt.bh.helux.nl/
ldap_search_base = dc=helux,dc=nl

ldap_id_use_start_tls = true
ldap_tls_cacert = /usr/local/share/ca-certificates/cacert.crt

cache_credentials = true
enumerate = true

cd /usr/local/share/ca-certificates
wget http://ldap.mngt.bh.helux.nl/cacert.crt
update-ca-certificates

/usr/share/pam-configs/mkhomedir

Name: activate mkhomedir
Default: yes
Priority: 900
Session-Type: Additional
Session:
  required  pam_mkhomedir.so umask=0022 skel=/etc/skel

pam-auth-update

Because of the LACP link the network is not available when I startup the server. It takes some time before everything is running. Samba wants to connect to LDAP and it fails when LDAP is not reachable. So this is a wait service.

/etc/systemd/system/network-wait-online.service

#
# Uses 'hostname --all-fqdns' to confirm that both: IP address[es] assigned, and DNS operational
#

[Unit]
Description=Wait for Network to be Online
Documentation=man:systemd.service(5) man:systemd.special(7)
Conflicts=shutdown.target
After=network.target
Before=network-online.target

[Service]
Type=oneshot
ExecStart=/bin/bash -c 'while [ -z $(hostname --all-fqdns) ]; do sleep 1; done'
TimeoutStartSec=1min 30s

[Install]
WantedBy=network-online.target

systemctl daemon-reload
systemctl enable network-wait-online

systemctl daemon-reload
systemctl enable network-wait-online

/etc/sysctl.d/10-local.conf

vm.swappiness=1

/etc/udev/rules.d/60-ssd-scheduler.rules

# set deadline scheduler for non-rotating disks
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="deadline"

apt install smartmontools

TODO: monitoring config with smartd and cron

update-alternatives --list python
update-alternatives: error: no alternatives for python

update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1
update-alternatives: using /usr/bin/python2.7 to provide /usr/bin/python (python) in auto mode

update-alternatives --install /usr/bin/python python /usr/bin/python3.5 2
update-alternatives: using /usr/bin/python3.5 to provide /usr/bin/python (python) in auto mode

apt install python3-pip python-pip

Now we can switch using the update-alternatives –config python. Remember pip for python 3 is pip3. pip is for python 2.7

Gather which disks are HDD and which are SDD.In my case:

1. sda: SSD (256GB)
2. sdb: HDD (4TB)
3. sdc: HDD (4TB)
4. sdd: SSD (120GB root disk)
5. sde: SSD (256GB)
6. sdf: HDD (4TB)
7. sdg: HDD (4TB)

I am going to create a RAID-1 mirror MD on the SSDs so I can enable writeback caching.

mdadm --create /dev/md0 --assume-clean --level=1 --raid-devices=2 /dev/sda /dev/sde

I am going to create a RAID-10 MD on the HDDs.

mdadm --create /dev/md1 --level=10 --raid-devices=4 /dev/sdb /dev/sdf /dev/sdc /dev/sdg

When rebooting the system md0 and 1 became md126 and md127. Also after rebooting the MD devices became read-only and pending sync. To enable de devices read-write:

mdadm --readwrite /dev/md126
mdadm --readwrite /dev/md127

apt install bcache-tools

In my case after a reboot /dev/md127 was the RAID-1 SSD mirror and /dev/md126 the RAID-10 HDD.

make-bcache --discard --writeback -C /dev/md127
make-bcache -B /dev/md126

We want to use UUIDs to avoid possible strange bootup behavior.

bcache-super-show /dev/md127 | grep cset.uuid
cset.uuid		f8d82c5e-a9a2-4b99-a2b0-15feab250372

Attach the SSD cache to the bcache0 device

echo f8d82c5e-a9a2-4b99-a2b0-15feab250372 > /sys/block/bcache0/bcache/attach

Enable writeback cache

echo writeback > /sys/block/bcache0/bcache/cache_mode

Show the status

cat /sys/block/bcache0/bcache/cache_mode
cat /sys/block/bcache0/bcache/state

echo writeback > /sys/block/bcache0/bcache/cache_mode

cat /sys/block/bcache0/bcache/cache_mode
cat /sys/block/bcache0/bcache/state

ls -l /sys/fs/bcache/
total 0
drwxr-xr-x 7 root root    0 Aug 12 21:28 af7bbe93-7bb8-4789-b879-a033c1331223
--w------- 1 root root 4096 Aug 12 21:25 register
--w------- 1 root root 4096 Aug 12 21:25 register_quiet

echo af7bbe93-7bb8-4789-b879-a033c1331223 > /sys/block/bcache0/bcache/detach

Wait for the cache status to become no cache. When attached the cache is dirty

watch -n 10 cat /sys/block/bcache0/bcache/state

echo 1 > /sys/block/md126/bcache/stop #HDD backing
echo 1 > /sys/block/md127/bcache/set/stop #SSD cache
echo 1 > /sys/block/bcache0/bcache/stop

apt install samba

/etc/samba/smb.conf

[global]
	realm = *
	workgroup = HELUX
	local master = No
	ldap admin dn = cn=manager,dc=helux,dc=nl
	ldap group suffix = ou=Group
	ldap idmap suffix = ou=Idmap
	ldap machine suffix = ou=Computer
	ldap passwd sync = yes
	ldap suffix = dc=helux,dc=nl
	ldap user suffix = ou=People
	load printers = No
	printcap name = cups
	max xmit = 32768
	min receivefile size = 2048
	server min protocol = NT1
	passdb backend = ldapsam:ldap://ldap.mngt.bh.helux.nl
	security = USER
	server role = standalone server
	deadtime = 15
	socket options = IPTOS_LOWDELAY SO_RCVBUF=131072 SO_SNDBUF=131072 TCP_NODELAY
	winbind enum groups = Yes
	winbind enum users = Yes
	ldapsam:editposix = yes
	ldapsam:trusted = yes
	idmap config * : backend = tdb
	admin users = root
	create mask = 0640
	directory mask = 0750
	aio read size = 2048
	aio write size = 2048
	use sendfile = Yes
	write cache size = 1024000


[home]
	path = /home/%U
	read only = No
	valid users = %U


[share]
	path = /volume1/share
	force create mode = 0644
	force directory mode = 0755
	read only = No
	valid users = %U

smbpaswd -W

Enter the LDAP manager password

apt install avahi-daemon avahi-utils

/etc/avahi/services/smb.service

<?xml version="1.0" standalone='no'?>
<!DOCTYPE service-group SYSTEM "avahi-service.dtd">
<service-group>
 <name replace-wildcards="yes">%h</name>
 <service>
   <type>_smb._tcp</type>
   <port>445</port>
 </service>
 <service>
   <type>_device-info._tcp</type>
   <port>0</port>
   <txt-record>model=RackMac</txt-record>
 </service>
</service-group>

Avahi daemon should start after samba

cp /lib/systemd/system/avahi.daemon.service /etc/systemd/system

/etc/systemd/system/avahi-daemon.service

[Unit]
.
.
Description=Avahi mDNS/DNS-SD Stack
Requires=avahi-daemon.socket
After=smbd.service
.
.

systemctl daemon-reload
systemctl enable avahi-daemon

Add the Docker CE repo and install from there

apt install apt-transport-https ca-certificates curl gnupg2 software-properties-common

curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -

add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"

apt install docker-ce

For Kolla

mkdir -p /etc/systemd/system/docker.service.d

tee /etc/systemd/system/docker.service.d/kolla.conf <<-'EOF'
[Service]
MountFlags=shared
EOF

systemctl daemon-reload

systemctl restart docker

echo configfs >/etc/modules-load.d/configfs.conf
systemctl status sys-kernel-config.mount