• Recent
    • Unsolved
    • Tags
    • Popular
    • Users
    • Groups
    • Search
    • Register
    • Login
    1. Home
    2. faboulous
    F
    • Profile
    • Following 0
    • Followers 0
    • Topics 1
    • Posts 4
    • Best 1
    • Controversial 0
    • Groups 0

    faboulous

    @faboulous

    1
    Reputation
    6
    Profile views
    4
    Posts
    0
    Followers
    0
    Following
    Joined Last Online

    faboulous Unfollow Follow

    Best posts made by faboulous

    • Ubuntu 20.04 NFS PXE autoinstall automation

      Here is a script to setup PXE boot for Ubuntu 20.04 server iso which can be preseed to automate server/desktop installation using cloud-init. It also automatically update fog pxe menu entry

      Since ubuntu 20, canonical deprecated debian-installer. Debian-installer is still working but I ran into issues with some statements and find out that using debian-installer successor cloud-init a bit easier to perform autoinstall.

      Ubuntu provide daily builds, we can use them to keep our iso updated by setting up a daily cronjob for this script and then add the user-data file needed as cloud-init input (inspired myself from this script which help greatly testing user-data file by wrapping it up into an ubuntu-server iso which can then be directly use either on vm or physical machine).

      #!/bin/bash
      
      DIST="ubuntu"
      RELEASE="focal"
      URL="https://cdimage.ubuntu.com/ubuntu-server/${RELEASE}/daily-live/current/${RELEASE}-live-server-amd64.iso"
      CHECKSUM_URL="https://cdimage.ubuntu.com/ubuntu-server/${RELEASE}/daily-live/current/SHA256SUMS"
      IMAGE_NAME="${DIST}/${RELEASE}"
      GIT_URL="https://some.git.repo.url/containing-user-data-file"
      
      # you need to setup an nfs server serving the NFS_DIR and a tftp server serving TFTP_DIR
      DOWNLOAD_DIR="/data/iso"
      TFTP_DIR="/tftpboot/os"
      NFS_DIR="/data/nfs"
      MOUNT_DIR="/media/tmp_image"
      NFS_SERVER_IP="your_fog_nfs_server_ip"
      
      # only define this if you have several vmlinuz or initrd inside the iso
      VMLINUZ=""
      INITRD=""
      
      # pxe parameters use in fog pxe menu entry 
      PXE_PARAMS="kernel tftp://\$\{fog-ip\}/os/${DIST}/${RELEASE}/vmlinuz
      initrd tftp://\$\{fog-ip\}/os/${DIST}/${RELEASE}/initrd
      imgargs boot=casper ip=dhcp netboot=nfs nfsroot=\$\{fog-ip\}:${NFS_DIR}/${IMAGE_NAME} rw"
      
      mkdir -p ${DOWNLOAD_DIR}/${DIST}/${RELEASE}
      image="${DOWNLOAD_DIR}/${DIST}/${RELEASE}/${RELEASE}-live-server-amd64.iso"
      
      function cleanup() {
              trap - SIGINT SIGTERM ERR EXIT
              if [ -n "${tmpdir+x}" ]; then
                      rm -rf "$tmpdir"
                      echo "🚽 Deleted temporary working directory $tmpdir"
              fi
      }
      
      trap cleanup SIGINT SIGTERM ERR EXIT
      
      testdir() {
        if [ ! -d "$1" ]; then
          echo "Directory $1 doesn't exist" 1>&2
          usage
          exit 1
        fi
      }
      
      checkIso() {
        if [ ! -f "${tmpdir}/SHA256SUMS-${today}" ]; then
          curl -NsSL ${CHECKSUM_URL} -o "${tmpdir}/SHA256SUMS-${today}"
        fi
        digest=$(sha256sum "${image}" | cut -f1 -d ' ')
        set +e
        grep -Fq "$digest" "${tmpdir}/SHA256SUMS-${today}"
        if [ $? -eq 0 ]; then
          echo "👍 Verification succeeded."
          set -e
          return 0
        else
          echo "👿 Verification of ISO digest failed." 1>&2;
          return 1
        fi
      }
      
      downloadIso() {
        if [ ! -f "${image}" ]; then
          echo "File $image doesn't exist, downloading iso"
          wget -P ${DOWNLOAD_DIR}/${DIST}/${RELEASE} ${URL}
        fi
        checkIso
        if [ $? -eq 1 ]; then
          rm ${image}
          downloadIso
        fi
      # TODO eventually check iso signature
      }
      
      # updating fog PXEMenu  
      updatePXEMenu() {
        echo "param $1 $2"
        pxeID=$(mysql --batch --skip-column-names --execute="select pxeID from fog.pxeMenu where pxeName=\"${DIST}_${RELEASE}_$1\"" | head -n 1)
        if [ -z "$pxeID"  ]; then
          echo "No entry for ${DIST} ${RELEASE} $1, generating a new one"
      
          mysql --batch --skip-column-names --execute="INSERT INTO fog.pxeMenu (pxeName, pxeDesc, pxeParams, pxeRegOnly, pxeArgs, pxeDefault, pxeHotKeyEnable, pxeKeySequence) VALUES (\"${DIST}_${RELEASE}_$1\", \"${DIST} ${RELEASE} $1 image\", '$2', 2, \"\", 0, \"0\", \"\");"
        else
          echo "Entry found for ${DIST} ${RELEASE}, updating"
          mysql --batch --skip-column-names --execute="UPDATE fog.pxeMenu SET pxeName=\"${DIST}_${RELEASE}_$1\", pxeDesc=\"${DIST} ${RELEASE} $1 image\", pxeParams='$2' where pxeID=${pxeID};"
        fi
      }
      
      # clone the git repo containing user-data files
      # it will search for user data store in /user-data/${DIST}/${RELEASE}/
      # you can them define as many user-data file as you want by creating storing 
      # them this way inside the repo : 
      # user-data/${DIST}/${RELEASE}/desktop-minimal/user-data
      # user-data/${DIST}/${RELEASE}/desktop-full/user-data 
      addUserData() {
        git clone --quiet ${GIT_URL} ${tmpdir}/cloud-init
        mkdir -p ${NFS_DIR}/${IMAGE_NAME}/nocloud
        cp -r ${tmpdir}/cloud-init/user-data/${DIST}/${RELEASE}/* ${NFS_DIR}/${IMAGE_NAME}/nocloud/
        for item in ${NFS_DIR}/${IMAGE_NAME}/nocloud/*; do
          item="$(echo $item | sed 's#.*/##')"
          tmp="${PXE_PARAMS} autoinstall ds=nocloud;s=/cdrom/nocloud/${item}"
          updatePXEMenu "autoinstall_${item}" "${tmp}"
        done
      }
      
      echo "PXE image script"
      if [ "$EUID" -ne 0 ]; then
        echo "Please run as root"
        exit
      fi
      
      tmpdir=$(mktemp -d)
      
      if [[ ! "$tmpdir" || ! -d "$tmpdir" ]]; then
              echo "💥 Could not create temporary working directory." 1>&2; exit 1;
      else
              echo "📁 Created temporary working directory $tmpdir"
      fi
      
      
      downloadIso
      mkdir -p ${MOUNT_DIR}
      echo "Trying to mount ${image} to ${MOUNT_DIR}..."
      mount -ro loop ${image} ${MOUNT_DIR}
      vmlinuzs=( $(find ${MOUNT_DIR} -name "vmlinuz*") )
      index=0
      nbelem=${#vmlinuzs[@]}
      if [ ${nbelem} -ne 1 ]; then
        echo "absent or multiple vmlinuz file, please run interactive script" 1>&2
        exit 1
      fi
      VMLINUZ=${vmlinuzs[${index}]}
      echo "The file ${VMLINUZ} will be used"
      initrds=( $(find ${MOUNT_DIR} -name "initrd*") )
      index=0
      nbelem=${#initrds[@]}
      if [ ${nbelem} -ne 1 ]; then
        echo "absent or multiple initrd file, please run interactive script" 1>&2
        exit 1
      fi
      INITRD=${initrds[${index}]}
      echo "The file ${INITRD} will be used"
      
      mkdir -vp ${TFTP_DIR}/${IMAGE_NAME}
      mkdir -vp ${NFS_DIR}/${IMAGE_NAME}
      echo "Copying the image file to NFS Server"
      cp -Rf ${MOUNT_DIR}/* ${NFS_DIR}/${IMAGE_NAME}
      # match hidden files too (store UUID needed to use NFS as it perform a check)
      cp -Rf ${MOUNT_DIR}/.[!.]* ${NFS_DIR}/${IMAGE_NAME}
      echo "Copying the vmlinuz & initrd files to TFTP Server"
      cp -f ${VMLINUZ} ${TFTP_DIR}/${IMAGE_NAME}/
      cp -f ${INITRD} ${TFTP_DIR}/${IMAGE_NAME}/
      umount ${MOUNT_DIR}
      
      echo "Editing TFTP config file"
      # setting up an entry with no autoinstall 
      updatePXEMenu "server" "${PXE_PARAMS}"
      # setting up one entry per user-data stored in your git repo 
      addUserData
      echo "Restarting TFTP Server"
      service tftpd-hpa restart
      echo "Finished"
      
      

      To setup manually this instead of using this script, you can follow this kind of instruction but perform the following command after this one cp -R /mnt/loop/* /images/os/mint/19.1

      cp -R /mnt/loop/.[!.]* /images/os/mint/19.1
      

      This script can be adapted to setup other distro iso (without automation as I think only ubuntu will work with cloud init)

      posted in Tutorials
      F
      faboulous

    Latest posts made by faboulous

    • Ubuntu 20.04 NFS PXE autoinstall automation

      Here is a script to setup PXE boot for Ubuntu 20.04 server iso which can be preseed to automate server/desktop installation using cloud-init. It also automatically update fog pxe menu entry

      Since ubuntu 20, canonical deprecated debian-installer. Debian-installer is still working but I ran into issues with some statements and find out that using debian-installer successor cloud-init a bit easier to perform autoinstall.

      Ubuntu provide daily builds, we can use them to keep our iso updated by setting up a daily cronjob for this script and then add the user-data file needed as cloud-init input (inspired myself from this script which help greatly testing user-data file by wrapping it up into an ubuntu-server iso which can then be directly use either on vm or physical machine).

      #!/bin/bash
      
      DIST="ubuntu"
      RELEASE="focal"
      URL="https://cdimage.ubuntu.com/ubuntu-server/${RELEASE}/daily-live/current/${RELEASE}-live-server-amd64.iso"
      CHECKSUM_URL="https://cdimage.ubuntu.com/ubuntu-server/${RELEASE}/daily-live/current/SHA256SUMS"
      IMAGE_NAME="${DIST}/${RELEASE}"
      GIT_URL="https://some.git.repo.url/containing-user-data-file"
      
      # you need to setup an nfs server serving the NFS_DIR and a tftp server serving TFTP_DIR
      DOWNLOAD_DIR="/data/iso"
      TFTP_DIR="/tftpboot/os"
      NFS_DIR="/data/nfs"
      MOUNT_DIR="/media/tmp_image"
      NFS_SERVER_IP="your_fog_nfs_server_ip"
      
      # only define this if you have several vmlinuz or initrd inside the iso
      VMLINUZ=""
      INITRD=""
      
      # pxe parameters use in fog pxe menu entry 
      PXE_PARAMS="kernel tftp://\$\{fog-ip\}/os/${DIST}/${RELEASE}/vmlinuz
      initrd tftp://\$\{fog-ip\}/os/${DIST}/${RELEASE}/initrd
      imgargs boot=casper ip=dhcp netboot=nfs nfsroot=\$\{fog-ip\}:${NFS_DIR}/${IMAGE_NAME} rw"
      
      mkdir -p ${DOWNLOAD_DIR}/${DIST}/${RELEASE}
      image="${DOWNLOAD_DIR}/${DIST}/${RELEASE}/${RELEASE}-live-server-amd64.iso"
      
      function cleanup() {
              trap - SIGINT SIGTERM ERR EXIT
              if [ -n "${tmpdir+x}" ]; then
                      rm -rf "$tmpdir"
                      echo "🚽 Deleted temporary working directory $tmpdir"
              fi
      }
      
      trap cleanup SIGINT SIGTERM ERR EXIT
      
      testdir() {
        if [ ! -d "$1" ]; then
          echo "Directory $1 doesn't exist" 1>&2
          usage
          exit 1
        fi
      }
      
      checkIso() {
        if [ ! -f "${tmpdir}/SHA256SUMS-${today}" ]; then
          curl -NsSL ${CHECKSUM_URL} -o "${tmpdir}/SHA256SUMS-${today}"
        fi
        digest=$(sha256sum "${image}" | cut -f1 -d ' ')
        set +e
        grep -Fq "$digest" "${tmpdir}/SHA256SUMS-${today}"
        if [ $? -eq 0 ]; then
          echo "👍 Verification succeeded."
          set -e
          return 0
        else
          echo "👿 Verification of ISO digest failed." 1>&2;
          return 1
        fi
      }
      
      downloadIso() {
        if [ ! -f "${image}" ]; then
          echo "File $image doesn't exist, downloading iso"
          wget -P ${DOWNLOAD_DIR}/${DIST}/${RELEASE} ${URL}
        fi
        checkIso
        if [ $? -eq 1 ]; then
          rm ${image}
          downloadIso
        fi
      # TODO eventually check iso signature
      }
      
      # updating fog PXEMenu  
      updatePXEMenu() {
        echo "param $1 $2"
        pxeID=$(mysql --batch --skip-column-names --execute="select pxeID from fog.pxeMenu where pxeName=\"${DIST}_${RELEASE}_$1\"" | head -n 1)
        if [ -z "$pxeID"  ]; then
          echo "No entry for ${DIST} ${RELEASE} $1, generating a new one"
      
          mysql --batch --skip-column-names --execute="INSERT INTO fog.pxeMenu (pxeName, pxeDesc, pxeParams, pxeRegOnly, pxeArgs, pxeDefault, pxeHotKeyEnable, pxeKeySequence) VALUES (\"${DIST}_${RELEASE}_$1\", \"${DIST} ${RELEASE} $1 image\", '$2', 2, \"\", 0, \"0\", \"\");"
        else
          echo "Entry found for ${DIST} ${RELEASE}, updating"
          mysql --batch --skip-column-names --execute="UPDATE fog.pxeMenu SET pxeName=\"${DIST}_${RELEASE}_$1\", pxeDesc=\"${DIST} ${RELEASE} $1 image\", pxeParams='$2' where pxeID=${pxeID};"
        fi
      }
      
      # clone the git repo containing user-data files
      # it will search for user data store in /user-data/${DIST}/${RELEASE}/
      # you can them define as many user-data file as you want by creating storing 
      # them this way inside the repo : 
      # user-data/${DIST}/${RELEASE}/desktop-minimal/user-data
      # user-data/${DIST}/${RELEASE}/desktop-full/user-data 
      addUserData() {
        git clone --quiet ${GIT_URL} ${tmpdir}/cloud-init
        mkdir -p ${NFS_DIR}/${IMAGE_NAME}/nocloud
        cp -r ${tmpdir}/cloud-init/user-data/${DIST}/${RELEASE}/* ${NFS_DIR}/${IMAGE_NAME}/nocloud/
        for item in ${NFS_DIR}/${IMAGE_NAME}/nocloud/*; do
          item="$(echo $item | sed 's#.*/##')"
          tmp="${PXE_PARAMS} autoinstall ds=nocloud;s=/cdrom/nocloud/${item}"
          updatePXEMenu "autoinstall_${item}" "${tmp}"
        done
      }
      
      echo "PXE image script"
      if [ "$EUID" -ne 0 ]; then
        echo "Please run as root"
        exit
      fi
      
      tmpdir=$(mktemp -d)
      
      if [[ ! "$tmpdir" || ! -d "$tmpdir" ]]; then
              echo "💥 Could not create temporary working directory." 1>&2; exit 1;
      else
              echo "📁 Created temporary working directory $tmpdir"
      fi
      
      
      downloadIso
      mkdir -p ${MOUNT_DIR}
      echo "Trying to mount ${image} to ${MOUNT_DIR}..."
      mount -ro loop ${image} ${MOUNT_DIR}
      vmlinuzs=( $(find ${MOUNT_DIR} -name "vmlinuz*") )
      index=0
      nbelem=${#vmlinuzs[@]}
      if [ ${nbelem} -ne 1 ]; then
        echo "absent or multiple vmlinuz file, please run interactive script" 1>&2
        exit 1
      fi
      VMLINUZ=${vmlinuzs[${index}]}
      echo "The file ${VMLINUZ} will be used"
      initrds=( $(find ${MOUNT_DIR} -name "initrd*") )
      index=0
      nbelem=${#initrds[@]}
      if [ ${nbelem} -ne 1 ]; then
        echo "absent or multiple initrd file, please run interactive script" 1>&2
        exit 1
      fi
      INITRD=${initrds[${index}]}
      echo "The file ${INITRD} will be used"
      
      mkdir -vp ${TFTP_DIR}/${IMAGE_NAME}
      mkdir -vp ${NFS_DIR}/${IMAGE_NAME}
      echo "Copying the image file to NFS Server"
      cp -Rf ${MOUNT_DIR}/* ${NFS_DIR}/${IMAGE_NAME}
      # match hidden files too (store UUID needed to use NFS as it perform a check)
      cp -Rf ${MOUNT_DIR}/.[!.]* ${NFS_DIR}/${IMAGE_NAME}
      echo "Copying the vmlinuz & initrd files to TFTP Server"
      cp -f ${VMLINUZ} ${TFTP_DIR}/${IMAGE_NAME}/
      cp -f ${INITRD} ${TFTP_DIR}/${IMAGE_NAME}/
      umount ${MOUNT_DIR}
      
      echo "Editing TFTP config file"
      # setting up an entry with no autoinstall 
      updatePXEMenu "server" "${PXE_PARAMS}"
      # setting up one entry per user-data stored in your git repo 
      addUserData
      echo "Restarting TFTP Server"
      service tftpd-hpa restart
      echo "Finished"
      
      

      To setup manually this instead of using this script, you can follow this kind of instruction but perform the following command after this one cp -R /mnt/loop/* /images/os/mint/19.1

      cp -R /mnt/loop/.[!.]* /images/os/mint/19.1
      

      This script can be adapted to setup other distro iso (without automation as I think only ubuntu will work with cloud init)

      posted in Tutorials
      F
      faboulous
    • RE: Preseeded (unattended) netboot UEFI Debian installation

      @george1421 I was asking for the suggestion made by @fogman4 which seems to be focused on Ubuntu, with a large amount or ram set for ramdisk

      posted in Tutorials
      F
      faboulous
    • RE: Preseeded (unattended) netboot UEFI Debian installation

      @fogman4 do machines need a large amount of ram booting this way? I have a lot of machines with low spec to install: 4G of ram which is not enough to unpack the iso using http method. More detail on what I already tried here :
      https://forums.fogproject.org/topic/14590/20-04-autoinstall

      posted in Tutorials
      F
      faboulous
    • RE: 20.04 autoinstall

      Hi I run into the same issue trying to boot a linuxmint 20 (and also ubuntu 20). After some digging, it appears that the Unable to find a live file system on the network show up because do_nfsmount() from the casper script chipped inside the initrd isn’t able to mount correctly the nfs share.

      I extracted (using unmkinitramfs) the nfsmount binary to test it standalone on a working nfs share using the same option provided by this casper script (nfsmount -o nolock -o ro ${NFSOPTS} ${NFSROOT} ${mountpoint}) which work well.

      I also tried to use the initrd from a linuxmint 19 to load the linuxmint 20 iso and… it worked but without any big surprise I ran into issue while I tried to install the os.

      The differences between mint19 and mint20 are some check performed after nfsmount call:

      do_nfsmount from mint 20

      do_nfsmount() {
          rc=1
          modprobe "${MP_QUIET}" nfs
          if [ -z "${NFSOPTS}" ]; then
              NFSOPTS=""
          else
              NFSOPTS=",${NFSOPTS}"
          fi
      
          [ "$quiet" != "y" ] && log_begin_msg "Trying nfsmount -o nolock -o ro ${NFSOPTS} ${NFSROOT} ${mountpoint}"
          # FIXME: This while loop is an ugly HACK round an nfs bug
          i=0
          while [ "$i" -lt 60 ]; do
              if nfsmount -o nolock -o ro${NFSOPTS} "${NFSROOT}" "${mountpoint}"; then
                  if is_casper_path $mountpoint && matches_uuid $mountpoint; then
                      rc=0
                  else
                      umount $mountpoint
                  fi
                  break
              fi
              sleep 1
              i="$(($i + 1))"
          done
          return ${rc}
      }
      

      do_nfsmount from mint 19

      do_nfsmount() {
          rc=1
          modprobe "${MP_QUIET}" nfs
          if [ -z "${NFSOPTS}" ]; then
              NFSOPTS=""
          else
              NFSOPTS=",${NFSOPTS}"
          fi
      
          [ "$quiet" != "y" ] && log_begin_msg "Trying nfsmount -o nolock -o ro ${NFSOPTS} ${NFSROOT} ${mountpoint}"
          # FIXME: This while loop is an ugly HACK round an nfs bug
          i=0
          while [ "$i" -lt 60 ]; do
              nfsmount -o nolock -o ro${NFSOPTS} "${NFSROOT}" "${mountpoint}" && rc=0 && break
              sleep 1
              i="$(($i + 1))"
          done
          return ${rc}
      }
      

      I tried to add some debug to identify what is going wrong but I couldn’t manage to “repack” the extracted initrd into a clean working one ending up on a kernel panic (I spend quite some time trying out, initrd archive structure seems to have recently evolve to chip intel and amd firmware inside). The only changed I made was adding some echo into the casper file. If someone manage to do so, it would be interesting to check what is_casper_path and matches_uuid does and return.

      As I wasn’t able to produce a custom initrd, I tried to configure a http server to serve the iso on similar way as suggested on the comment bellow by @pacman366. But ubuntu/mint desktop iso are quite big (2GB) and 4GB ram isn’t enough to be able to extract the iso. I also tried to boot from http following these instructions: https://www.plop.at/en/ploplinux/live/networkboot-linux.html#pxel61 without any success. I’m not sure how | in the url work (no request received on the nginx web server).

      I’d really like to be able to fix this, any idea/help would be appreciated 🙂

      posted in General Problems
      F
      faboulous