Ganesha & NFS Installation
Using Ubuntu 18.04
Ganesha
apt -y install nfs-ganesha-gluster
apt-get install nfs-ganesha-vfs Mention that, must to install nfs-ganesha-vfs for local volume (non-glustserfs) testing.
Still not yet trying hook with glusterfs.
cat /etc/ganesha/ganesha.conf
EXPORT{
Export_Id = 1 ; # Export ID unique to each export
#Path = "volume_path"; # Path of the volume to be exported. Eg: "/test_volume"
Pseudo = /vfs_distributed;
Path = /vol_distributed;
#FSAL {
# name = GLUSTER;
# hostname = "10.xx.xx.xx"; # IP of one of the nodes in the trusted pool
# volume = "volume_name"; # Volume name. Eg: "test_volume"
#}
Access_type = RW; # Access permissions
Squash = No_root_squash; # To enable/disable root squashing
Disable_ACL = TRUE; # To enable/disable ACL
# Pseudo = "pseudo_path"; # NFSv4 pseudo path for this export. Eg: "/test_volume_pseudo"
Protocols = "3","4" ; # NFS protocols supported
Transports = "UDP","TCP" ; # Transport protocols supported
#SecType = "sys"; # Security flavors supported
SecType = "sys,krb5,krb5i,krb5p";
FSAL {
Name = VFS;
}
}Create Volume and set permission
mkdir /vol_distributed
chown 777 -R /vol_distributed
mkdir /vfs_distributed
chown 777 -R /vfs_distributed
Start rpcbind
service rpcbind startReload Ganesha
systemctl restart nfs-ganesha Detect mount point
root@ganesha:~# showmount -e localhost
Export list for localhost:
/vol_distributed (everyone)Client Mount.
mount -t nfs -o vers=3 nfsserver:/vol_distributed ./test -vvvvTrouble Shooting
access denied by server while mounting
install
apt-get install nfs-ganesha-vfs showmount -e localhost
apt-get install nfs-ganesha-vfs client idle on write
chown 777 -R /vol_distributed
chown 777 -R /vfs_distributedNFS-Kernel Installation
Install nfs-kernel
sudo apt-get install nfs-kernel-server
edit and show export points
root@nfs:~# cat /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
# to NFS clients. See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4 gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
///srv/nfs 172.16.155.0/24(sync,no_subtree_check,insecure,rw)
/srv/nfs *(sync,no_subtree_check,insecure,rw,no_root_squash)Set directory permission
mkdir /srv/nfs
chmod 777 /srv/nfs
Allow all or Allow some subnet to connect to.
sudo exportfs -raReload NFS
sudo systemctl restart nfs-kernel-serverdetect mount point
root@nfs:~# showmount -e localhost
Export list for localhost:
/srv/nfs *
Client mount by using v3 or v4.
mount -t nfs -o vers=3 nfsserver:/srv/nfs /mnt -vvvv
mount -t nfs -o vers=4 nfsserver:/srv/nfs /mnt -vvvvNow it support v4.
if we want to disable v4 or v3
/etc/default/nfs-kernel-server
//RPCMOUNTDOPTS="--manage-gids --no-nfs-version 3"
//RPCMOUNTDOPTS="--manage-gids --no-nfs-version 4"Docker Version Ganesha
see the reference
https://github.com/mitcdh/docker-nfs-ganesha/blob/master/Dockerfile
Dockerfile
#FROM 192.168.51.130:5000/ebotubuntu:v0.2
#FROM ubuntu:18.04
FROM jrei/systemd-ubuntu:18.04
# Environment Setting
# working directory
RUN apt-get update
RUN apt -y install nfs-ganesha-gluster
RUN apt-get install nfs-ganesha-vfs -y
RUN apt-get install netbase -y
COPY ganesha.conf /etc/ganesha/ganesha.conf
RUN mkdir /vol_distributed
RUN chown 777 -R /vol_distributed
RUN mkdir /vfs_distributed
RUN chown 777 -R /vfs_distributed
#RUN mkdir -p /run/rpcbind /export /var/run/dbus
#RUN touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr
#RUN chmod 755 /run/rpcbind/*
#RUN chown messagebus:messagebus /var/run/dbus
EXPOSE 111 111/udp 662 2049 38465-38467
#ENTRYPOINT ["../entrypoint.sh"]
#ENTRYPOINT ["../run.sh"]
#CMD ["bash","./run.sh"]
#EXPOSE 1337
#CMD ["systemctl", "start", "nfs-ganesha.service"]root@kubecontext:/opt/mount/buildimage/ganesha# cat ganesha.conf
EXPORT{
Export_Id = 1 ; # Export ID unique to each export
#Path = "volume_path"; # Path of the volume to be exported. Eg: "/test_volume"
Pseudo = /vfs_distributed;
Path = /vol_distributed;
#FSAL {
# name = GLUSTER;
# hostname = "10.xx.xx.xx"; # IP of one of the nodes in the trusted pool
# volume = "volume_name"; # Volume name. Eg: "test_volume"
#}
Access_type = RW; # Access permissions
Squash = No_root_squash; # To enable/disable root squashing
Disable_ACL = TRUE; # To enable/disable ACL
# Pseudo = "pseudo_path"; # NFSv4 pseudo path for this export. Eg: "/test_volume_pseudo"
Protocols = "3","4" ; # NFS protocols supported
Transports = "UDP","TCP" ; # Transport protocols supported
#SecType = "sys"; # Security flavors supported
SecType = "sys,krb5,krb5i,krb5p";
FSAL {
Name = VFS;
}
}docker run -d --privileged --cap-add SYS_ADMIN -v /sys/fs/cgroup:/sys/fs/cgroup:ro dockerepo:5566/nfsganisha:testinitialize, Local mount, and check result.
showmount -e localhost
mount -t nfs -o vers=4 172.17.0.2:/vol_distributed ./test -vvvv
Docker Version NFS
Dockerfile
#FROM 192.168.51.130:5000/ebotubuntu:v0.2
#FROM ubuntu:18.04
FROM jrei/systemd-ubuntu:18.04
# Environment Setting
# working directory
RUN apt-get update
RUN apt-get install nfs-kernel-server -y
COPY exports /etc/exports
RUN mkdir -p /srv/nfs
RUN chmod 777 /srv/nfs
#RUN exportfs -ra //no need to export since it restarted
#EXPOSE 111 111/udp 662 2049 38465-38467
#ENTRYPOINT ["../entrypoint.sh"]
#ENTRYPOINT ["../run.sh"]
#CMD ["bash","./run.sh"]
#EXPOSE 1337
#CMD ["systemctl", "start", "nfs-ganesha.service"]
#CMD ["sleep", "inf"]cat /etc/exports
/srv/nfs *(sync,fsid=0,no_subtree_check,insecure,rw,no_root_squash)
Adding fsid attribute to /etc/export.
docker run -d --privileged --cap-add SYS_ADMIN -v /sys/fs/cgroup:/sys/fs/cgroup:ro dockerrepo:5566/nfs:testneed to mount /sys/fs/cgroup to container or the systemd will be failed.
showmount -e localhost
mount -t nfs -o vers=4 172.17.0.2:/srv/nfs test -vvvv
//mount.nfs nfs-access:/srv/nfs test -o nolock
Either use '-o nolock' to keep locks local, or start statd. adding
-o nolockin mount
rpcinfo -p localhost
program vers proto port service
100000 4 tcp 111 portmapper
100000 3 tcp 111 portmapper
100000 2 tcp 111 portmapper
100000 4 udp 111 portmapper
100000 3 udp 111 portmapper
100000 2 udp 111 portmapper
100005 1 udp 32767 mountd
100005 1 tcp 32767 mountd
100005 2 udp 32767 mountd
100005 2 tcp 32767 mountd
100005 3 udp 32767 mountd
100005 3 tcp 32767 mountd
100003 3 tcp 2049 nfs
100003 4 tcp 2049 nfs
100227 3 tcp 2049
100003 3 udp 2049 nfs
100227 3 udp 2049
100021 1 udp 46271 nlockmgr
100021 3 udp 46271 nlockmgr
100021 4 udp 46271 nlockmgr
100021 1 tcp 33029 nlockmgr
100021 3 tcp 33029 nlockmgr
100021 4 tcp 33029 nlockmgr
It will show how the port expose to outside, it is very useful for k8s service used. ( we just fix some port here).
Useful Docker Command
Remove all images
docker images| awk '{print $3}'|xargs docker rmi -Remove all container
docker ps -a| awk '{print $1}' |xargs docker rm -f -Kubernetes Deployment
apiVersion: v1
kind: Service
metadata:
name: nfs-access
labels:
app: nfs-access
spec:
ports:
- name: http
protocol: TCP
#port is loadbalancer port
port: 8500
# for clustering, port=targetPort
# adding clusterIP: none for setting up deep dns
clusterIP: None
selector:
role: nfs-server
---
apiVersion: v1
kind: Service
metadata:
name: ext-nfs
labels:
app: ext-nfs
spec:
externalIPs:
- 172.16.155.188
ports:
- name: nfs
port: 2049
- name: mountd
port: 32767
- name: rpcbind
port: 111
- name: nolock
port: 34623
- name: nolock1
port: 37419
selector:
role: nfs-server
---
kind: Service
apiVersion: v1
metadata:
name: nfs-server
spec:
ports:
- name: nfs
port: 2049
- name: mountd
port: 32767
- name: rpcbind
port: 111
- name: nolock
port: 34623
- name: nolock1
port: 37419
selector:
role: nfs-server
---
apiVersion: v1
kind: ReplicationController
metadata:
name: nfs-server
spec:
replicas: 1
selector:
role: nfs-server
template:
metadata:
labels:
role: nfs-server
spec:
#nodeSelector:
# external-storage: "true"
containers:
- name: nfs-server
#image: gcr.io/google_containers/volume-nfs:0.8
image: 172.16.155.136:5000/nfs:test
ports:
- name: nfs
containerPort: 2049
- name: mountd
containerPort: 32767
- name: rpcbind
containerPort: 111
- name: nolock
containerPort: 34623
- name: nolock1
containerPort: 37419
securityContext:
privileged: true
capabilities:
add:
- NET_ADMIN
volumeMounts:
#- mountPath: /exports
- mountPath: /srv/nfs
name: nfs-export-fast
- mountPath: /sys/fs/cgroup
name: systemd-mount
volumes:
- name: nfs-export-fast
hostPath:
path: /data/nfs
- name: systemd-mount
hostPath:
path: /sys/fs/cgroup
note: use
nfs-accessto client access.
Client in K8S
apt-get install nfs-common
mkdir test
mount.nfs nfs-access:/srv/nfs test -o nolockand make sure yaml contains privileged enabling setting.
securityContext:
privileged: true
capabilities:
add:
- NET_ADMIN