Ganesha

Ganesha & NFS Installation

Using Ubuntu 18.04

Ganesha

apt -y install nfs-ganesha-gluster
apt-get install nfs-ganesha-vfs

Mention that, must to install nfs-ganesha-vfs for local volume (non-glustserfs) testing. Still not yet trying hook with glusterfs.

cat /etc/ganesha/ganesha.conf

EXPORT{
    Export_Id = 1 ;   # Export ID unique to each export
    #Path = "volume_path";  # Path of the volume to be exported. Eg: "/test_volume"
    Pseudo = /vfs_distributed;
    Path = /vol_distributed;

    #FSAL {
    #    name = GLUSTER;
    #    hostname = "10.xx.xx.xx";  # IP of one of the nodes in the trusted pool
    #    volume = "volume_name";  # Volume name. Eg: "test_volume"
    #}

    Access_type = RW;    # Access permissions
    Squash = No_root_squash; # To enable/disable root squashing
    Disable_ACL = TRUE;  # To enable/disable ACL
   # Pseudo = "pseudo_path";  # NFSv4 pseudo path for this export. Eg: "/test_volume_pseudo"
    Protocols = "3","4" ;    # NFS protocols supported
    Transports = "UDP","TCP" ; # Transport protocols supported
    #SecType = "sys";     # Security flavors supported
    SecType = "sys,krb5,krb5i,krb5p";
    FSAL {
           Name = VFS;
    }
}

Create Volume and set permission

mkdir /vol_distributed
chown 777 -R  /vol_distributed
mkdir /vfs_distributed
chown 777 -R /vfs_distributed

Start rpcbind

service rpcbind start

Reload Ganesha

systemctl restart nfs-ganesha

Detect mount point

root@ganesha:~# showmount -e localhost
Export list for localhost:
/vol_distributed (everyone)

Client Mount.

mount -t nfs -o vers=3 nfsserver:/vol_distributed ./test -vvvv

Trouble Shooting

access denied by server while mounting

install

apt-get install nfs-ganesha-vfs

showmount -e localhost

apt-get install nfs-ganesha-vfs

client idle on write

chown 777 -R  /vol_distributed
chown 777 -R /vfs_distributed

NFS-Kernel Installation

Install nfs-kernel

    sudo apt-get install nfs-kernel-server

edit and show export points

root@nfs:~# cat /etc/exports
# /etc/exports: the access control list for filesystems which may be exported
#       to NFS clients.  See exports(5).
#
# Example for NFSv2 and NFSv3:
# /srv/homes       hostname1(rw,sync,no_subtree_check) hostname2(ro,sync,no_subtree_check)
#
# Example for NFSv4:
# /srv/nfs4        gss/krb5i(rw,sync,fsid=0,crossmnt,no_subtree_check)
# /srv/nfs4/homes  gss/krb5i(rw,sync,no_subtree_check)

///srv/nfs  172.16.155.0/24(sync,no_subtree_check,insecure,rw)
/srv/nfs  *(sync,no_subtree_check,insecure,rw,no_root_squash)

Set directory permission

mkdir /srv/nfs
chmod 777 /srv/nfs

Allow all or Allow some subnet to connect to.

sudo exportfs -ra

Reload NFS

sudo systemctl restart nfs-kernel-server

detect mount point


root@nfs:~# showmount -e localhost
Export list for localhost:
/srv/nfs *

Client mount by using v3 or v4.

mount -t nfs -o vers=3 nfsserver:/srv/nfs /mnt  -vvvv
mount -t nfs -o vers=4 nfsserver:/srv/nfs /mnt  -vvvv

Now it support v4.

if we want to disable v4 or v3

/etc/default/nfs-kernel-server

//RPCMOUNTDOPTS="--manage-gids --no-nfs-version 3"
//RPCMOUNTDOPTS="--manage-gids --no-nfs-version 4"

Docker Version Ganesha

see the reference

https://github.com/mitcdh/docker-nfs-ganesha/blob/master/Dockerfile

Dockerfile

#FROM 192.168.51.130:5000/ebotubuntu:v0.2
#FROM ubuntu:18.04
FROM jrei/systemd-ubuntu:18.04

# Environment Setting
# working directory
RUN apt-get update
RUN apt -y install nfs-ganesha-gluster
RUN apt-get install nfs-ganesha-vfs  -y
RUN apt-get install netbase  -y

COPY ganesha.conf /etc/ganesha/ganesha.conf
RUN mkdir /vol_distributed
RUN chown 777 -R  /vol_distributed
RUN mkdir /vfs_distributed
RUN chown 777 -R /vfs_distributed
#RUN mkdir -p /run/rpcbind /export /var/run/dbus
#RUN touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr
#RUN chmod 755 /run/rpcbind/*
#RUN chown messagebus:messagebus /var/run/dbus


EXPOSE 111 111/udp 662 2049 38465-38467

#ENTRYPOINT ["../entrypoint.sh"]
#ENTRYPOINT ["../run.sh"]
#CMD ["bash","./run.sh"]
#EXPOSE 1337
#CMD ["systemctl", "start", "nfs-ganesha.service"]

root@kubecontext:/opt/mount/buildimage/ganesha# cat ganesha.conf
EXPORT{
    Export_Id = 1 ;   # Export ID unique to each export
    #Path = "volume_path";  # Path of the volume to be exported. Eg: "/test_volume"
    Pseudo = /vfs_distributed;
    Path = /vol_distributed;

    #FSAL {
    #    name = GLUSTER;
    #    hostname = "10.xx.xx.xx";  # IP of one of the nodes in the trusted pool
    #    volume = "volume_name";  # Volume name. Eg: "test_volume"
    #}

    Access_type = RW;    # Access permissions
    Squash = No_root_squash; # To enable/disable root squashing
    Disable_ACL = TRUE;  # To enable/disable ACL
   # Pseudo = "pseudo_path";  # NFSv4 pseudo path for this export. Eg: "/test_volume_pseudo"
    Protocols = "3","4" ;    # NFS protocols supported
    Transports = "UDP","TCP" ; # Transport protocols supported
    #SecType = "sys";     # Security flavors supported
    SecType = "sys,krb5,krb5i,krb5p";
    FSAL {
           Name = VFS;
    }
}

docker run -d --privileged --cap-add SYS_ADMIN -v /sys/fs/cgroup:/sys/fs/cgroup:ro dockerepo:5566/nfsganisha:test

initialize, Local mount, and check result.


showmount -e localhost
mount -t nfs -o vers=4 172.17.0.2:/vol_distributed ./test -vvvv

Docker Version NFS

Dockerfile

#FROM 192.168.51.130:5000/ebotubuntu:v0.2
#FROM ubuntu:18.04
FROM jrei/systemd-ubuntu:18.04

# Environment Setting
# working directory
RUN apt-get update
RUN apt-get install nfs-kernel-server -y
COPY exports /etc/exports
RUN mkdir -p /srv/nfs
RUN chmod 777 /srv/nfs
#RUN exportfs -ra //no need to export since it restarted

#EXPOSE 111 111/udp 662 2049 38465-38467
#ENTRYPOINT ["../entrypoint.sh"]
#ENTRYPOINT ["../run.sh"]
#CMD ["bash","./run.sh"]
#EXPOSE 1337
#CMD ["systemctl", "start", "nfs-ganesha.service"]
#CMD ["sleep", "inf"]

cat /etc/exports

/srv/nfs  *(sync,fsid=0,no_subtree_check,insecure,rw,no_root_squash)

Adding fsid attribute to /etc/export.

docker run -d --privileged --cap-add SYS_ADMIN -v /sys/fs/cgroup:/sys/fs/cgroup:ro dockerrepo:5566/nfs:test

need to mount /sys/fs/cgroup to container or the systemd will be failed.

showmount -e localhost
mount -t nfs -o vers=4 172.17.0.2:/srv/nfs test -vvvv
//mount.nfs nfs-access:/srv/nfs test -o nolock

Either use '-o nolock' to keep locks local, or start statd. adding -o nolock in mount

rpcinfo -p localhost

   program vers proto   port  service
    100000    4   tcp    111  portmapper
    100000    3   tcp    111  portmapper
    100000    2   tcp    111  portmapper
    100000    4   udp    111  portmapper
    100000    3   udp    111  portmapper
    100000    2   udp    111  portmapper
    100005    1   udp  32767  mountd
    100005    1   tcp  32767  mountd
    100005    2   udp  32767  mountd
    100005    2   tcp  32767  mountd
    100005    3   udp  32767  mountd
    100005    3   tcp  32767  mountd
    100003    3   tcp   2049  nfs
    100003    4   tcp   2049  nfs
    100227    3   tcp   2049
    100003    3   udp   2049  nfs
    100227    3   udp   2049
    100021    1   udp  46271  nlockmgr
    100021    3   udp  46271  nlockmgr
    100021    4   udp  46271  nlockmgr
    100021    1   tcp  33029  nlockmgr
    100021    3   tcp  33029  nlockmgr
    100021    4   tcp  33029  nlockmgr

It will show how the port expose to outside, it is very useful for k8s service used. ( we just fix some port here).

Useful Docker Command

Remove all images

docker images| awk '{print $3}'|xargs docker rmi  -

Remove all container

docker ps -a| awk '{print $1}' |xargs docker rm -f -

Kubernetes Deployment

apiVersion: v1
kind: Service
metadata:
  name: nfs-access
  labels:
    app: nfs-access
spec:
  ports:
  - name: http
    protocol: TCP
    #port is loadbalancer port
    port: 8500
    # for clustering, port=targetPort
    # adding clusterIP: none for setting up deep dns
  clusterIP: None
  selector:
    role: nfs-server
---
apiVersion: v1
kind: Service
metadata:
    name: ext-nfs
    labels:
      app: ext-nfs
spec:
  externalIPs:
    - 172.16.155.188
  ports:
    - name: nfs
      port: 2049
    - name: mountd
      port: 32767
    - name: rpcbind
      port: 111
    - name: nolock
      port: 34623
    - name: nolock1
      port: 37419
  selector:
    role: nfs-server
---
kind: Service
apiVersion: v1
metadata:
  name: nfs-server
spec:
  ports:
    - name: nfs
      port: 2049
    - name: mountd
      port: 32767
    - name: rpcbind
      port: 111
    - name: nolock
      port: 34623
    - name: nolock1
      port: 37419
  selector:
    role: nfs-server
---
apiVersion: v1
kind: ReplicationController
metadata:
  name: nfs-server
spec:
  replicas: 1
  selector:
    role: nfs-server
  template:
    metadata:
      labels:
        role: nfs-server
    spec:
      #nodeSelector:
      #  external-storage: "true"
      containers:
      - name: nfs-server
        #image: gcr.io/google_containers/volume-nfs:0.8
        image: 172.16.155.136:5000/nfs:test
        ports:
          - name: nfs
            containerPort: 2049
          - name: mountd
            containerPort: 32767
          - name: rpcbind
            containerPort: 111
          - name: nolock
            containerPort: 34623
          - name: nolock1
            containerPort: 37419
        securityContext:
          privileged: true
          capabilities:
             add:
               - NET_ADMIN
        volumeMounts:
          #- mountPath: /exports
          - mountPath: /srv/nfs
            name: nfs-export-fast
          - mountPath: /sys/fs/cgroup
            name: systemd-mount
      volumes:
        - name: nfs-export-fast
          hostPath:
            path: /data/nfs
        - name: systemd-mount
          hostPath:
            path: /sys/fs/cgroup

note: use nfs-access to client access.

Client in K8S

apt-get install nfs-common
mkdir test
mount.nfs nfs-access:/srv/nfs test -o nolock

and make sure yaml contains privileged enabling setting.

        securityContext:
          privileged: true
          capabilities:
             add:
               - NET_ADMIN

瞧那部落格

Thursday, December 27, 2018

nfs in k8s

Ganesha & NFS Installation

Ganesha

Trouble Shooting

access denied by server while mounting

showmount -e localhost

client idle on write

NFS-Kernel Installation

Docker Version Ganesha

Docker Version NFS

Useful Docker Command

Kubernetes Deployment

Client in K8S

About Me

Followers