Omnia beegfs issue#621

roles/beegfs_role/defaults/main.yml

@@ -0,0 +1,126 @@
+#  Copyright 2021 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+# Option to enable/disable services
+beegfs_enable:
+  mgmt: false
+  meta: false
+  oss: false
+  admon: false
+  client: false
+  tuning: false
+
+# Can be present or absent
+beegfs_state: present
+
+# Kernel modules to ensure are loaded prior to BeeGFS deploy/startup
+beegfs_kmod_preload: []
+
+# Network interfaces in order of preference
+# (leaving empty means InfiniBand & RDMA enabled devices are preferred)
+beegfs_interfaces: []
+
+# Distribution-specific parameters
+beegfs_distro_vars:
+  RedHat:
+    beegfs_repo_url: "https://www.beegfs.io/release/latest-stable/dists/beegfs-rhel8.repo"
+    beegfs_repo_key: "https://www.beegfs.io/release/latest-stable/gpg/RPM-GPG-KEY-beegfs"
+    beegfs_repo_file: "/etc/yum.repos.d/beegfs-rhel8.repo"
+    kernel_dev_pkg: "kernel-devel"
+    rdma_dev_pkgs:
+      - "librdmacm"
+      - "libibmad"
+      - "libibumad"
+      - "rdma-core"
+      - "libibverbs"
+      - "libibverbs-utils"
+      - "libbeegfs-ib"
+
+  Rocky:
+    beegfs_repo_url: "https://www.beegfs.io/release/latest-stable/dists/rocky"
+    beegfs_repo_key: "https://www.beegfs.io/release/latest-stable/gpg/RPM-GPG-KEY-beegfs"
+    beegfs_repo_file: "/etc/yum.repos.d/beegfs-rocky.repo"
+    kernel_dev_pkg: "kernel-devel"
+    rdma_dev_pkgs:
+      - "librdmacm"
+      - "libibmad"
+      - "libibumad"
+      - "rdma-core"
+      - "libibverbs"
+      - "libibverbs-utils"
+      - "libbeegfs-ib"
+
+  Debian:
+    beegfs_repo_url: "https://www.beegfs.io/release/latest-stable/dists/beegfs-deb9.list"
+    beegfs_repo_key: "https://www.beegfs.io/release/latest-stable/gpg/DEB-GPG-KEY-beegfs"
+    beegfs_repo_file: "/etc/apt/sources.list.d/beegfs_deb9.list"
+    kernel_dev_pkg: "linux-headers"
+    rdma_dev_pkgs:
+      - "librdmacm-dev"
+      - "libibmad-dev"
+      - "libibumad-dev"
+      - "libibverbs-dev"
+      - "libbeegfs-ib"
+
+# BeeGFS repo
+beegfs_add_repos: true
+beegfs_repo_url: "{{ beegfs_distro_vars[ansible_os_family]['beegfs_repo_url'] }}"
+beegfs_repo_key: "{{ beegfs_distro_vars[ansible_os_family]['beegfs_repo_key'] }}"
+beegfs_repo_file: "{{ beegfs_distro_vars[ansible_os_family]['beegfs_repo_file'] }}"
+
+# Define to enable support for BeeGFS data transport over RDMA (IB, OPA, RoCE)
+beegfs_rdma: true
+
+# Default filesystem options
+beegfs_filesystem_opts: "-K -d su=128k,sw=8 -l version=2,su=128k -isize=512"
+beegfs_mount_opts: "noatime,nodiratime,logbufs=8,logbsize=256k,largeio,inode64,swalloc,allocsize=131072k,nobarrier"
+beegfs_force_format: false
+beegfs_fstype: "xfs"
+
+# Storage server options
+beegfs_oss_path_prefix: "/data/beegfs/beegfs_oss"
+beegfs_oss:
+  - port: 8003
+    dev: "/dev/sdb"
+    path:  # specifying dev overrides path
+beegfs_oss_tunable: "{{ beegfs_oss | selectattr('dev', 'defined') | map(attribute='dev') | map('relpath', '/dev/' ) | list }}"
+# Whether to support multiple ports per OSS:
+# - if this is set to false, make sure that you configure the ports to the same
+#   value when passing beegfs_oss as an input.
+beegfs_oss_multi_port: true
+
+# Management options
+beegfs_mgmt_path: "/data/beegfs/beegfs_mgmtd"
+beegfs_mgmt_host:  # Example: "{{ groups['cluster_beegfs_mgmt'] | first }}"
+
+# Metadata storage options
+beegfs_meta_path: "/data/beegfs/beegfs_meta"
+beegfs_meta_dev:  # /dev/sdb
+beegfs_meta_fstype:  # Example: "ext4"
+
+# Client mount configs
+beegfs_client: []
+# Each item in the beegfs_client list should look like this:
+# - path: "/mnt/beegfs" # If there are multiple clients, use different path for each client.
+#   port: 8004 # If there are multiple clients, use different port for each client.
+#   mgmt_host: # Use this to override the default beegfs_mgmt_host in cases there are multiple clients.
+
+# Should the package manager use the `present` or `latest` flag for
+# BeeGFS? This is useful for upgrading.
+beegfs_update: false
+beegfs_package_action: "{{ 'latest' if beegfs_update | bool else 'present' }}"
+
+# Path to the kernel module
+beegfs_kernel_module: "/lib/modules/{{ ansible_kernel }}/updates/fs/beegfs_autobuild/beegfs.ko"
+...

roles/beegfs_role/files/beegfs-client.service.d/override.conf

@@ -0,0 +1,7 @@
+# The reason for this change is to enable beegfs-client.service to persist
+# through reboots in hyperconverged setup where the client and the oss server is
+# running node and the client is not allowed to mount until oss server is ready.
+[Service]
+Type=forking
+Restart=on-failure
+RestartSec=5s

roles/beegfs_role/files/beegfs-oss-tuning.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=BeeGFS Tuning Service
+Documentation=http://www.beegfs.com/content/documentation/
+Requires=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/bin/bash /opt/beegfs/beegfs-oss-tuning.sh
+Type=oneshot
+RemainAfterExit=1
+
+[Install]
+WantedBy=multi-user.target

roles/beegfs_role/handlers/main.yml

@@ -0,0 +1,69 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+- name: Restart BeeGFS mgmt service
+  systemd:
+    name: beegfs-mgmtd
+    daemon-reload: true
+    state: restarted
+  become: true
+  when: beegfs_enable.mgmt | bool
+
+- name: Restart BeeGFS meta service
+  systemd:
+    name: beegfs-meta
+    daemon-reload: true
+    state: restarted
+  become: true
+  when: beegfs_enable.meta | bool
+
+- name: Restart BeeGFS storage service
+  systemd:
+    name: "beegfs-storage@{{ item.port }}"
+    daemon-reload: true
+    state: restarted
+  become: true
+  with_items: "{{ beegfs_oss }}"
+  when: beegfs_enable.oss | bool
+
+- name: Restart BeeGFS admon service
+  systemd:
+    name: beegfs-admon
+    daemon-reload: true
+    state: restarted
+  become: true
+  when: beegfs_enable.admon | bool
+
+- name: Restart BeeGFS client service
+  systemd:
+    name: "{{ item }}"
+    daemon-reload: true
+    state: restarted
+  become: true
+  with_items:
+    - beegfs-helperd
+    - beegfs-client
+  when: beegfs_enable.client | bool
+  retries: 3
+  delay: 3
+  register: result
+  until: result is success
+
+- name: Restart BeeGFS tuning service
+  systemd:
+    name: beegfs-oss-tuning
+    daemon-reload: true
+    state: restarted
+  become: true
+...

roles/beegfs_role/meta/main.yml

@@ -0,0 +1,44 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+galaxy_info:
+  role_name: beegfs
+  author: Dell Technologies
+  description: >
+    This role provisions an exisiting cluster to support Beegfs
+    management, metadata, object storage server and client roles.
+  company: Dell Technologies
+  license: Apache2
+  min_ansible_version: 2.5.2
+  platforms:
+    - name: EL
+      versions:
+        - 7
+        - 8
+    - name: Rocky
+      versions:
+        - 8
+    - name: Debian
+      versions:
+        - "stretch"
+    - name: Ubuntu
+      versions:
+        - "xenial"
+        - "bionic"
+  galaxy_tags:
+    - cloud
+    - storage
+    - beegfs
+
+dependencies: []

roles/beegfs_role/tasks/admon.yml

@@ -0,0 +1,15 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+...

roles/beegfs_role/tasks/client.yml

@@ -0,0 +1,119 @@
+#  Copyright 2020 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+- name: Install packages for BeeGFS client
+  package:
+    name: "{{ item }}"
+    state: "{{ beegfs_package_action }}"
+  with_items:
+    - beegfs-client
+    - beegfs-helperd
+    - beegfs-utils
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Define client kernel build for IB
+  lineinfile:
+    path: "/etc/beegfs/beegfs-client-autobuild.conf"
+    regexp: "^buildArgs="
+    line: "{{ item.line }}"
+  when: item.condition
+  with_items:
+    - { line: "buildArgs=-j8 BEEGFS_OPENTK_IBVERBS=1", condition: "{{ beegfs_rdma | bool }}" }
+    - { line: "buildArgs=-j8", condition: "{{ not beegfs_rdma | bool }}" }
+  notify: Restart BeeGFS client service
+  become: true
+
+- name: Ensure kernel development headers are present
+  package:
+    name: "{{ beegfs_distro_vars[ansible_os_family]['kernel_dev_pkg'] }}-{{ ansible_kernel }}"
+    state: present
+  notify: Restart BeeGFS client service
+  become: true
+
+- name: Ensure gcc is installed
+  package:
+    name: "gcc"
+    state: present
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Rebuild the BeeGFS client kernel module
+  command: /etc/init.d/beegfs-client rebuild
+  args:
+    creates: "{{ beegfs_kernel_module }}"
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Ensure the BeeGFS mount point exists
+  file:
+    path: "{{ client_path }}"
+    state: directory
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Copy over beegfs-mounts config file
+  template:
+    src: beegfs-mounts.conf.j2
+    dest: /etc/beegfs/beegfs-mounts.conf
+    mode: 0644
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Make of copy of BeeGFS client config file if it doesn't exist
+  copy:
+    remote_src: true
+    src: /etc/beegfs/beegfs-client.conf
+    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    force: false
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Configure beegfs-client config file to point to the specified management host
+  lineinfile:
+    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    regexp: "^sysMgmtdHost"
+    line: "sysMgmtdHost = {{ client_mgmt_host }}"
+  become: true
+
+- name: Configure beegfs-client config file to use the specified port
+  lineinfile:
+    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    regexp: "^connClientPortUDP"
+    line: "connClientPortUDP = {{ client_port }}"
+  become: true
+
+- name: Configure beegfs-client config file to ensure the specifed connection interface is used
+  lineinfile:
+    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    regexp: "^connInterfacesFile"
+    line: "connInterfacesFile = /etc/beegfs/connInterfacesFile"
+  become: true
+
+- name: Copy default beegfs-client config file to enable beegfs_ctl in the management host
+  copy:
+    remote_src: true
+    dest: /etc/beegfs/beegfs-client.conf
+    src: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    force: true
+  when: client_mgmt_host == inventory_hostname
+  become: true
+
+- name: Copy beegfs-client.service.d/ to /etc/systemd/system/
+  copy:
+    src: beegfs-client.service.d/
+    dest: /etc/systemd/system/beegfs-client.service.d/
+  notify: Restart BeeGFS client service
+  become: true
+...