2017-06-28 15:53 UTC

View Issue Details Jump to Notes ]
IDProjectCategoryView StatusLast Update
0012426CentOS-7grub2public2017-03-28 17:06
Reportersz1305 
PrioritynormalSeveritymajorReproducibilityalways
StatusnewResolutionopen 
Platformx86_64OSCentOS7OS Versionall versions
Product Version 
Target VersionFixed in Version 
Summary0012426: system boot hangs after lvm snapshot from root partition
DescriptionI'm using LVM snapshots for backups on CentOS7. After create a snapshot from root partition, the system hangs on next start (all versions).

To fix this problem it is possible to boot rescue option (grub2 boot menu) and then remove the snapshot from the root partition. After a reboot the system is ok (but without a snapshot based restore option).

The Issuse was also discussed in CentOS-Forum
https://www.centos.org/forums/viewtopic.php?f=47&t=59006

thanks ...
TagsNo tags attached.
abrt_hash
URL
Attached Files
  • txt file icon screenlog.txt (9,954 bytes) 2017-03-25 10:59 -
    cat /etc/centos-release
    CentOS Linux release 7.3.1611 (Core) 
    
    ...
    
    kroot@test01:~\[root@test01 ~]# cat /etc/os-release 
    NAME="CentOS Linux"
    VERSION="7 (Core)"
    ID="centos"
    ID_LIKE="rhel fedora"
    VERSION_ID="7"
    PRETTY_NAME="CentOS Linux 7 (Core)"
    ANSI_COLOR="0;31"
    CPE_NAME="cpe:/o:centos:centos:7"
    HOME_URL="https://www.centos.org/"
    BUG_REPORT_URL="https://bugs.centos.org/"
    
    CENTOS_MANTISBT_PROJECT="CentOS-7"
    CENTOS_MANTISBT_PROJECT_VERSION="7"
    REDHAT_SUPPORT_PRODUCT="centos"
    REDHAT_SUPPORT_PRODUCT_VERSION="7"
    
    ...
    
    kroot@test01:~\[root@test01 ~]# cat /etc/fstab 
    
    #
    # /etc/fstab
    # Created by anaconda on Sat Mar 25 10:38:22 2017
    #
    # Accessible filesystems, by reference, are maintained under '/dev/disk'
    # See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
    #
    /dev/mapper/vgsys-root  /                       ext4    defaults        1 1
    UUID=d3fc978a-5990-449e-a0f5-81c5e87bd3f2 /boot                   ext4    defaults        1 2
    /dev/mapper/vgsys-home  /home                   ext4    defaults        1 2
    /dev/mapper/vgsys-tmp   /tmp                    ext4    defaults        1 2
    /dev/mapper/vgsys-var   /var                    ext4    defaults        1 2
    /dev/mapper/vgsys-swap  swap                    swap    defaults        0 0
    
    ...
    
    kroot@test01:~\[root@test01 ~]# cat /etc/boot/grub2/grub.cfg 
    #
    # DO NOT EDIT THIS FILE
    #
    # It is automatically generated by grub2-mkconfig using templates
    # from /etc/grub.d and settings from /etc/default/grub
    #
    
    ### BEGIN /etc/grub.d/00_header ###
    set pager=1
    
    if [ -s $prefix/grubenv ]; then
      load_env
    fi
    if [ "${next_entry}" ] ; then
       set default="${next_entry}"
       set next_entry=
       save_env next_entry
       set boot_once=true
    else
       set default="${saved_entry}"
    fi
    
    if [ x"${feature_menuentry_id}" = xy ]; then
      menuentry_id_option="--id"
    else
      menuentry_id_option=""
    fi
    
    export menuentry_id_option
    
    if [ "${prev_saved_entry}" ]; then
      set saved_entry="${prev_saved_entry}"
      save_env saved_entry
      set prev_saved_entry=
      save_env prev_saved_entry
      set boot_once=true
    fi
    
    function savedefault {
      if [ -z "${boot_once}" ]; then
        saved_entry="${chosen}"
        save_env saved_entry
      fi
    }
    
    function load_video {
      if [ x$feature_all_video_module = xy ]; then
        insmod all_video
      else
        insmod efi_gop
        insmod efi_uga
        insmod ieee1275_fb
        insmod vbe
        insmod vga
        insmod video_bochs
        insmod video_cirrus
      fi
    }
    
    terminal_output console
    if [ x$feature_timeout_style = xy ] ; then
      set timeout_style=menu
      set timeout=5
    # Fallback normal timeout code in case the timeout_style feature is
    # unavailable.
    else
      set timeout=5
    fi
    ### END /etc/grub.d/00_header ###
    
    ### BEGIN /etc/grub.d/00_tuned ###
    set tuned_params=""
    ### END /etc/grub.d/00_tuned ###
    
    ### BEGIN /etc/grub.d/01_users ###
    if [ -f ${prefix}/user.cfg ]; then
      source ${prefix}/user.cfg
      if [ -n "${GRUB2_PASSWORD}" ]; then
        set superusers="root"
        export superusers
        password_pbkdf2 root ${GRUB2_PASSWORD}
      fi
    fi
    ### END /etc/grub.d/01_users ###
    
    ### BEGIN /etc/grub.d/10_linux ###
    menuentry 'CentOS Linux (3.10.0-514.10.2.el7.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-514.el7.x86_64-advanced-70ba10c1-ec9a-454b-aebf-6669971d1ade' {
    	load_video
    	set gfxpayload=keep
    	insmod gzio
    	insmod part_msdos
    	insmod ext2
    	set root='hd0,msdos1'
    	if [ x$feature_platform_search_hint = xy ]; then
    	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	else
    	  search --no-floppy --fs-uuid --set=root d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	fi
    	linux16 /vmlinuz-3.10.0-514.10.2.el7.x86_64 root=/dev/mapper/vgsys-root ro crashkernel=auto rd.lvm.lv=vgsys/root rd.lvm.lv=vgsys/swap rhgb quiet LANG=de_DE.UTF-8
    	initrd16 /initramfs-3.10.0-514.10.2.el7.x86_64.img
    }
    menuentry 'CentOS Linux (3.10.0-514.el7.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-514.el7.x86_64-advanced-70ba10c1-ec9a-454b-aebf-6669971d1ade' {
    	load_video
    	set gfxpayload=keep
    	insmod gzio
    	insmod part_msdos
    	insmod ext2
    	set root='hd0,msdos1'
    	if [ x$feature_platform_search_hint = xy ]; then
    	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	else
    	  search --no-floppy --fs-uuid --set=root d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	fi
    	linux16 /vmlinuz-3.10.0-514.el7.x86_64 root=/dev/mapper/vgsys-root ro crashkernel=auto rd.lvm.lv=vgsys/root rd.lvm.lv=vgsys/swap rhgb quiet LANG=de_DE.UTF-8
    	initrd16 /initramfs-3.10.0-514.el7.x86_64.img
    }
    menuentry 'CentOS Linux (0-rescue-da292c99689e43bc8fb941b0462a1078) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-da292c99689e43bc8fb941b0462a1078-advanced-70ba10c1-ec9a-454b-aebf-6669971d1ade' {
    	load_video
    	insmod gzio
    	insmod part_msdos
    	insmod ext2
    	set root='hd0,msdos1'
    	if [ x$feature_platform_search_hint = xy ]; then
    	  search --no-floppy --fs-uuid --set=root --hint-bios=hd0,msdos1 --hint-efi=hd0,msdos1 --hint-baremetal=ahci0,msdos1 --hint='hd0,msdos1'  d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	else
    	  search --no-floppy --fs-uuid --set=root d3fc978a-5990-449e-a0f5-81c5e87bd3f2
    	fi
    	linux16 /vmlinuz-0-rescue-da292c99689e43bc8fb941b0462a1078 root=/dev/mapper/vgsys-root ro crashkernel=auto rd.lvm.lv=vgsys/root rd.lvm.lv=vgsys/swap rhgb quiet
    	initrd16 /initramfs-0-rescue-da292c99689e43bc8fb941b0462a1078.img
    }
    
    ### END /etc/grub.d/10_linux ###
    
    ### BEGIN /etc/grub.d/20_linux_xen ###
    ### END /etc/grub.d/20_linux_xen ###
    
    ### BEGIN /etc/grub.d/20_ppc_terminfo ###
    ### END /etc/grub.d/20_ppc_terminfo ###
    
    ### BEGIN /etc/grub.d/30_os-prober ###
    ### END /etc/grub.d/30_os-prober ###
    
    ### BEGIN /etc/grub.d/40_custom ###
    # This file provides an easy way to add custom menu entries.  Simply type the
    # menu entries you want to add after this comment.  Be careful not to change
    # the 'exec tail' line above.
    ### END /etc/grub.d/40_custom ###
    
    ### BEGIN /etc/grub.d/41_custom ###
    if [ -f  ${config_directory}/custom.cfg ]; then
      source ${config_directory}/custom.cfg
    elif [ -z "${config_directory}" -a -f  $prefix/custom.cfg ]; then
      source $prefix/custom.cfg;
    fi
    ### END /etc/grub.d/41_custom ###
    
    ...
    
    kroot@test01:~\[root@test01 ~]# lvdisplay 
      --- Logical volume ---
      LV Path                /dev/vgsys/var
      LV Name                var
      VG Name                vgsys
      LV UUID                EG0dfi-bwdj-sQMv-lHBV-U7e4-jF1k-QbbTTW
      LV Write Access        read/write
      LV Creation host, time test01.maindom.local, 2017-03-25 10:38:20 +0100
      LV Status              available
      # open                 1
      LV Size                1,46 GiB
      Current LE             375
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           253:2
       
      --- Logical volume ---
      LV Path                /dev/vgsys/home
      LV Name                home
      VG Name                vgsys
      LV UUID                3ZeyZD-GwKf-14MI-xgnc-IsKH-AJq5-duqnee
      LV Write Access        read/write
      LV Creation host, time test01.maindom.local, 2017-03-25 10:38:20 +0100
      LV Status              available
      # open                 1
      LV Size                1,95 GiB
      Current LE             500
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           253:3
       
      --- Logical volume ---
      LV Path                /dev/vgsys/root
      LV Name                root
      VG Name                vgsys
      LV UUID                Bz5uLE-5pw3-Fwll-fIwW-dZIw-lChD-WsJS7c
      LV Write Access        read/write
      LV Creation host, time test01.maindom.local, 2017-03-25 10:38:21 +0100
      LV Status              available
      # open                 1
      LV Size                6,84 GiB
      Current LE             1750
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           253:0
       
      --- Logical volume ---
      LV Path                /dev/vgsys/tmp
      LV Name                tmp
      VG Name                vgsys
      LV UUID                OpbLTS-Hdxl-LnyI-xlre-ODzE-bWcy-iXilzh
      LV Write Access        read/write
      LV Creation host, time test01.maindom.local, 2017-03-25 10:38:22 +0100
      LV Status              available
      # open                 1
      LV Size                1,46 GiB
      Current LE             375
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           253:5
       
      --- Logical volume ---
      LV Path                /dev/vgsys/swap
      LV Name                swap
      VG Name                vgsys
      LV UUID                WMNqr3-1V9h-3FZV-OOWS-duwK-Kpbw-8wlWi5
      LV Write Access        read/write
      LV Creation host, time test01.maindom.local, 2017-03-25 10:38:22 +0100
      LV Status              available
      # open                 2
      LV Size                1,95 GiB
      Current LE             500
      Segments               1
      Allocation             inherit
      Read ahead sectors     auto
      - currently set to     8192
      Block device           253:1
       
    ...
       
    kroot@test01:~\[root@test01 ~]# lvcreate --snapshot --size 200m --name root_2017-03-25_11-00 /dev/root/vgsys/root 
      Using default stripesize 64,00 KiB.
      Logical volume "root_2017-03-25_11-00" created.
    kroot@test01:~\[root@test01 ~]# 
    txt file icon screenlog.txt (9,954 bytes) 2017-03-25 10:59 +
  • png file icon Screenshot.png (194,180 bytes) 2017-03-25 11:18 -
    png file icon Screenshot.png (194,180 bytes) 2017-03-25 11:18 +
  • txt file icon CentOS_LVM_ERROR.txt (16,266 bytes) 2017-03-27 18:11 -
    Q. Are you creating snapshot by the given command line, or using some automated tool?
    A: command line (example): lvcreate --snapshot --size 200m --name root_2017-03-27_10-19 /dev/vgsys/root
    
    Q. Is your system's "systemd" and "dracut" package updated?
    A: Yes
    [root@test01 ~]# rpm -qa | grep systemd
    systemd-libs-219-30.el7_3.7.x86_64
    systemd-219-30.el7_3.7.x86_64
    systemd-sysv-219-30.el7_3.7.x86_64
    
    [root@test01 ~]# rpm -qa | grep dracut
    dracut-033-463.el7.x86_64
    dracut-network-033-463.el7.x86_64
    dracut-config-rescue-033-463.el7.x86_64
    
    ***
    
    Q. Does "rpm -qV systemd dracut kernel" report any weird errors?
    A: No
    [root@test01 ~]# rpm -qV systemd dracut kernel
    [root@test01 ~]#
    
    ***
    
    Q. Does removing "LANG=de_DE.UTF-8" from kernel option change things?
    A: No, see below section TESTING
    
    Q. Does using snapshot name withOUT hyphen (use snap_root_YYYY_mm_dd) change things?
    A: No, see below section TESTING
    
    Q. How did you partition the machine? VG name "vgsys" isn't a default.
    A: 
    [root@test01 ~]# fdisk -l
    Disk /dev/sda: 53.7 GB, 53687091200 bytes, 104857600 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    Disk label type: dos
    Disk identifier: 0x000b67f2
    
       Gerät  boot.     Anfang        Ende     Blöcke   Id  System
    /dev/sda1   *        2048     1026047      512000   83  Linux
    /dev/sda2         1026048    30738431    14856192   8e  Linux LVM
    
    Disk /dev/mapper/vgsys-root: 7340 MB, 7340032000 bytes, 14336000 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    
    
    Disk /dev/mapper/vgsys-swap: 2097 MB, 2097152000 bytes, 4096000 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    
    
    Disk /dev/mapper/vgsys-var: 1572 MB, 1572864000 bytes, 3072000 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    
    
    Disk /dev/mapper/vgsys-home: 2097 MB, 2097152000 bytes, 4096000 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    
    
    Disk /dev/mapper/vgsys-tmp: 1572 MB, 1572864000 bytes, 3072000 sectors
    Units = Sektoren of 1 * 512 = 512 bytes
    Sector size (logical/physical): 512 bytes / 512 bytes
    I/O size (minimum/optimal): 512 bytes / 512 bytes
    
    ***
    
    Q. Is physical partition MBR or GPT type?
    A: MBR
    
    ***
    
    ==============================================
    Section Testing
    ==============================================
    1) edit /boot/grub2/grub.cfg and changed kernel parameters (see below)
    - without "LANG=de_DE.UTF-8"
    - with "rd.retry=20"
    - with "rd.break=initqueue"
    
    kernel parameters in grub.cfg:
    linux16 /vmlinuz-3.10.0-514.10.2.el7.x86_64 root=/dev/mapper/vgsys-root ro crashkernel=auto rd.lvm.lv=vgsys/root rd.lvm.lv=vgsys/swap rd.retry=20 rd.break=initqueue
    
    2) create snapshot
    [root@test01 ~]# lvcreate --snapshot --size 200m --name snap /dev/vgsys/root
      Using default stripesize 64,00 KiB.
      Logical volume "snap" created.
    
    3) reboot
    ... and the machine hangs
    
    
    4) initqueue:/# output commands
    initqueue:/# ls -lR /usr/lib/dracut/hooks/initqueue/
    /usr/lib/dracut/hooks/initqueue/:
    total 4
    drwxr-xr-x 2 root 0   0 Mar 27 15:31 finished
    drwxr-xr-x 2 root 0   0 Mar 25 08:46 online
    drwxr-xr-x 2 root 0   0 Mar 27 15:24 settled
    -rw-r--r-- 1 root 0 145 Mar 27 15:31 setup_net_ens33.sh
    drwxr-xr-x 2 root 0   0 Mar 27 15:24 timeout
    
    /usr/lib/dracut/hooks/initqueue/finished:
    total 16
    -rw-r--r-- 1 root 0 32 Mar 27 15:24 devexists-\x2fdev\x2fmapper\x2fvgsys-root.sh
    -rw-r--r-- 1 root 0 25 Mar 27 15:24 devexists-\x2fdev\x2fvgsys\x2froot.sh
    -rw-r--r-- 1 root 0 25 Mar 27 15:24 devexists-\x2fdev\x2fvgsys\x2fswap.sh
    -rw-r--r-- 1 root 0 32 Mar 27 15:31 dhclient-ens33.sh
    
    /usr/lib/dracut/hooks/initqueue/online:
    total 0
    
    /usr/lib/dracut/hooks/initqueue/settled:
    total 4
    -rw-r--r-- 1 root 0 48 Mar 27 15:24 lvm_scan.sh
    
    /usr/lib/dracut/hooks/initqueue/timeout:
    total 8
    -rw-r--r-- 1 root 0   58 Mar 27 15:24 51-lvm_scan.sh
    -rwxr-xr-x 1 root 0 1352 Nov 11 18:27 99-rootfallback.sh
    
    ***
    
    initqueue:/# sh -x /sbin/lvm_scan
    + extraargs=
    + type getarg
    + . /lib/dracut-lib.sh
    ++ export DRACUT_SYSTEMD
    ++ export NEWROOT
    ++ '[' -n /sysroot ']'
    ++ '[' -d /sysroot ']'
    ++ '[' -d /run/initramfs ']'
    ++ '[' -d /run/lock ']'
    ++ '[' -d /run/log ']'
    ++ '[' -z 1 ']'
    ++ setdebug
    ++ '[' -f /etc/initrd-release ']'
    ++ '[' -z no ']'
    ++ debug_on
    ++ '[' no = yes ']'
    ++ hookdir=/lib/dracut/hooks
    ++ export hookdir
    ++ command -v findmnt
    ++ command -v pidof
    ++ setmemdebug
    ++ '[' -z 0 ']'
    ++ getargs rd.lvm.vg -d rd_LVM_VG=
    ++ debug_off
    ++ set +x
    + VGS=
    ++ getargs rd.lvm.lv -d rd_LVM_LV=
    ++ debug_off
    ++ set +x
    + LVS='vgsys/root vgsys/swap'
    ++ getargs rd.lvm.snapshot -d rd_LVM_SNAPSHOT=
    ++ debug_off
    ++ set +x
    + SNAPSHOT=
    ++ getargs rd.lvm.snapsize -d rd_LVM_SNAPSIZE=
    ++ debug_off
    ++ set +x
    + SNAPSIZE=
    + '[' -d /etc/lvm ']'
    ++ for f in '/tmp/.lvm_scan-*'
    ++ '[' -e /tmp/.lvm_scan-sda2 ']'
    ++ printf %s 'sda2 '
    + lvmdevs='sda2 '
    + '[' '!' -e /etc/lvm/lvm.conf ']'
    + OLDIFS=' 	
    '
    + IFS=.
    ++ lvm version
    + set '  LVM version:     2' 02 '166(2)-RHEL7 (2016-11-16)
      Library version: 1' 02 '135-RHEL7 (2016-11-16)
      Driver version:  4' 34 0
    + IFS=' 	
    '
    + maj='     2'
    + min=02
    + sub='166(2)-RHEL7'
    + sub=166
    + check_lvm_ver 2 2 57 2 02 166
    + maj=2
    + min=2
    + ver=57
    + '[' 2 -lt 2 ']'
    + '[' 2 -gt 2 ']'
    + '[' 02 -lt 2 ']'
    + '[' 02 -gt 2 ']'
    + '[' 166 -ge 57 ']'
    + return 0
    + nopoll='--poll n'
    + check_lvm_ver 2 2 65 2 2 166
    + maj=2
    + min=2
    + ver=65
    + '[' 2 -lt 2 ']'
    + '[' 2 -gt 2 ']'
    + '[' 2 -lt 2 ']'
    + '[' 2 -gt 2 ']'
    + '[' 166 -ge 65 ']'
    + return 0
    + sysinit=' --sysinit '
    + unset extraargs
    + export LVM_SUPPRESS_LOCKING_FAILURE_MESSAGES=1
    + LVM_SUPPRESS_LOCKING_FAILURE_MESSAGES=1
    + '[' -n '' ']'
    + '[' -n 'vgsys/root vgsys/swap' ']'
    + info 'Scanning devices sda2  for LVM logical volumes vgsys/root vgsys/swap'
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + lvm lvscan --ignorelockingfailure
    + vinfo
    + read line
    + info inactive ''\''/dev/vgsys/var'\''' '[1.46' 'GiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info inactive ''\''/dev/vgsys/home'\''' '[1.95' 'GiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info inactive Original ''\''/dev/vgsys/root'\''' '[6.84' 'GiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info inactive ''\''/dev/vgsys/tmp'\''' '[1.46' 'GiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info ACTIVE ''\''/dev/vgsys/swap'\''' '[1.95' 'GiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info inactive Snapshot ''\''/dev/vgsys/snap'\''' '[200.00' 'MiB]' inherit
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + '[' -n '' ']'
    + for LV in '$LVS'
    + '[' -z ' --sysinit ' ']'
    + lvm lvchange --yes -ay --sysinit vgsys/root
    + vinfo
    + read line
    + info modprobe: FATAL: Module dm-snapshot not found.
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info /usr/sbin/modprobe failed: 1
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + info 'Can'\''t' process LV vgsys/snap: snapshot target support missing from 'kernel?'
    + check_quiet
    + '[' -z yes ']'
    + '[' yes '!=' yes ']'
    + read line
    + '[' -n '' ']'
    + for LV in '$LVS'
    + '[' -z ' --sysinit ' ']'
    + lvm lvchange --yes -ay --sysinit vgsys/swap
    + vinfo
    + read line
    + '[' -n '' ']'
    + '[' -z 'vgsys/root vgsys/swap' -o -n '' ']'
    + '[' '' ']'
    + unset lvmwritten
    + udevadm settle
    + need_shutdown
    
    ***
    
    initqueue:/# lvm lvs
      LV   VG    Attr       LSize   Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
      home vgsys -wi-------   1.95g                                                    
      root vgsys owi---s---   6.84g                                                    
      snap vgsys swi---s--- 200.00m      root                                          
      swap vgsys -wi-a-----   1.95g                                                    
      tmp  vgsys -wi-------   1.46g                                                    
      var  vgsys -wi-------   1.46g                                                    
    
    ***
      
    initqueue:/# lvm vgdisplay
      --- Volume group ---
      VG Name               vgsys
      System ID             
      Format                lvm2
      Metadata Areas        1
      Metadata Sequence No  14
      VG Access             read/write
      VG Status             resizable
      MAX LV                0
      Cur LV                6
      Open LV               0
      Max PV                0
      Cur PV                1
      Act PV                1
      VG Size               14.16 GiB
      PE Size               4.00 MiB
      Total PE              3626
      Alloc PE / Size       3550 / 13.87 GiB
      Free  PE / Size       76 / 304.00 MiB
      VG UUID               bx5Tt7-9Iiz-vxt2-EEID-lD4j-NOM3-4zjq3A
    
    ***
      
    initqueue:/# lvm pvdisplay
      --- Physical volume ---
      PV Name               /dev/sda2
      VG Name               vgsys
      PV Size               14.17 GiB / not usable 4.00 MiB
      Allocatable           yes 
      PE Size               4.00 MiB
      Total PE              3626
      Free PE               76
      Allocated PE          3550
      PV UUID               JI3F6q-s9QC-fA28-KT13-zO8H-iiEr-0fkaoh
    
    ***
    
    initqueue:/# ls -lR /dev/disk
    /dev/disk:
    total 0
    drwxr-xr-x 2 root 0 120 Mar 27 15:43 by-id
    drwxr-xr-x 2 root 0 120 Mar 27 15:24 by-path
    drwxr-xr-x 2 root 0  80 Mar 27 15:43 by-uuid
    
    /dev/disk/by-id:
    total 0
    lrwxrwxrwx 1 root 0  9 Mar 27 15:24 ata-VMware_Virtual_SATA_CDRW_Drive_01000000000000000001 -> ../../sr0
    lrwxrwxrwx 1 root 0 10 Mar 27 15:43 dm-name-vgsys-swap -> ../../dm-0
    lrwxrwxrwx 1 root 0 10 Mar 27 15:43 dm-uuid-LVM-bx5Tt79Iizvxt2EEIDlD4jNOM34zjq3AWMNqr31V9h3FZVOOWSduwKKpbw8wlWi5 -> ../../dm-0
    lrwxrwxrwx 1 root 0 10 Mar 27 15:24 lvm-pv-uuid-JI3F6q-s9QC-fA28-KT13-zO8H-iiEr-0fkaoh -> ../../sda2
    
    /dev/disk/by-path:
    total 0
    lrwxrwxrwx 1 root 0  9 Mar 27 15:24 pci-0000:00:10.0-scsi-0:0:0:0 -> ../../sda
    lrwxrwxrwx 1 root 0 10 Mar 27 15:24 pci-0000:00:10.0-scsi-0:0:0:0-part1 -> ../../sda1
    lrwxrwxrwx 1 root 0 10 Mar 27 15:24 pci-0000:00:10.0-scsi-0:0:0:0-part2 -> ../../sda2
    lrwxrwxrwx 1 root 0  9 Mar 27 15:24 pci-0000:02:05.0-ata-2.0 -> ../../sr0
    
    /dev/disk/by-uuid:
    total 0
    lrwxrwxrwx 1 root 0 10 Mar 27 15:43 a1716299-9216-48e3-aa97-c02bfab26fb6 -> ../../dm-0
    lrwxrwxrwx 1 root 0 10 Mar 27 15:24 d3fc978a-5990-449e-a0f5-81c5e87bd3f2 -> ../../sda1
    
    ***
    
    initqueue:/# ls -lR /dev/mapper
    /dev/mapper:
    total 0
    crw------- 1 root 0 10, 236 Mar 27 15:24 control
    lrwxrwxrwx 1 root 0       7 Mar 27 15:43 vgsys-swap -> ../dm-0
    
    ***
    
    initqueue:/# cat /proc/modules
    ext4 583541 1 - Live 0xffffffffa026b000
    mbcache 14958 1 ext4, Live 0xffffffffa0082000
    jbd2 102945 1 ext4, Live 0xffffffffa0250000
    sr_mod 22416 0 - Live 0xffffffffa0158000
    cdrom 42556 1 sr_mod, Live 0xffffffffa0244000
    ata_generic 12910 0 - Live 0xffffffffa00a5000
    pata_acpi 13038 0 - Live 0xffffffffa0195000
    sd_mod 46322 3 - Live 0xffffffffa0237000
    crc_t10dif 12714 1 sd_mod, Live 0xffffffffa0024000
    crct10dif_generic 12647 0 - Live 0xffffffffa0227000
    crct10dif_pclmul 14289 1 - Live 0xffffffffa01e6000
    crct10dif_common 12595 3 crc_t10dif,crct10dif_generic,crct10dif_pclmul, Live 0xffffffffa0232000
    crc32c_intel 22079 0 - Live 0xffffffffa01df000
    serio_raw 13462 0 - Live 0xffffffffa01da000
    vmwgfx 235043 1 - Live 0xffffffffa01ec000
    drm_kms_helper 146456 1 vmwgfx, Live 0xffffffffa0170000
    syscopyarea 12529 1 drm_kms_helper, Live 0xffffffffa016b000
    sysfillrect 12701 1 drm_kms_helper, Live 0xffffffffa0164000
    sysimgblt 12640 1 drm_kms_helper, Live 0xffffffffa015f000
    fb_sys_fops 12703 1 drm_kms_helper, Live 0xffffffffa0094000
    ahci 34042 0 - Live 0xffffffffa014e000
    libahci 31992 1 ahci, Live 0xffffffffa008b000
    e1000 137544 0 - Live 0xffffffffa01b7000
    ata_piix 35038 0 - Live 0xffffffffa01ad000
    mptspi 22542 2 - Live 0xffffffffa0123000
    scsi_transport_spi 30732 1 mptspi, Live 0xffffffffa01a4000
    mptscsih 40150 1 mptspi, Live 0xffffffffa0143000
    mptbase 105960 2 mptspi,mptscsih, Live 0xffffffffa0108000
    ttm 93908 1 vmwgfx, Live 0xffffffffa012b000
    drm 372540 4 vmwgfx,drm_kms_helper,ttm, Live 0xffffffffa00ac000
    i2c_core 40756 2 drm_kms_helper,drm, Live 0xffffffffa009a000
    libata 247095 5 ata_generic,pata_acpi,ahci,libahci,ata_piix, Live 0xffffffffa0044000
    fjes 32197 0 - Live 0xffffffffa003b000
    dm_mirror 22135 0 - Live 0xffffffffa0030000
    dm_region_hash 20862 1 dm_mirror, Live 0xffffffffa001d000
    dm_log 18411 2 dm_mirror,dm_region_hash, Live 0xffffffffa002a000
    dm_mod 114430 4 dm_mirror,dm_log, Live 0xffffffffa0000000
    
    ***
    
    initqueue:/# sed -e '/^[\t ]*#/d; /^$/d' /etc/lvm/lvm.conf	;#contents of /etc/lvm/lvm.conf
    config {
    	checks = 1
    	abort_on_errors = 0
    	profile_dir = "/etc/lvm/profile"
    }
    devices {
    	dir = "/dev"
    	scan = [ "/dev" ]
    	obtain_device_list_from_udev = 1
    	external_device_info_source = "none"
    	preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ]
    	cache_dir = "/etc/lvm/cache"
    	cache_file_prefix = ""
    	write_cache_state = 1
    	sysfs_scan = 1
    	multipath_component_detection = 1
    	md_component_detection = 1
    	fw_raid_component_detection = 0
    	md_chunk_alignment = 1
    	data_alignment_detection = 1
    	data_alignment = 0
    	data_alignment_offset_detection = 1
    	ignore_suspended_devices = 0
    	ignore_lvm_mirrors = 1
    	disable_after_error_count = 0
    	require_restorefile_with_uuid = 1
    	pv_min_size = 2048
    	issue_discards = 0
    	allow_changes_with_duplicate_pvs = 1
    }
    allocation {
    	maximise_cling = 1
    	use_blkid_wiping = 1
    	wipe_signatures_when_zeroing_new_lvs = 1
    	mirror_logs_require_separate_pvs = 0
    	cache_pool_metadata_require_separate_pvs = 0
    	thin_pool_metadata_require_separate_pvs = 0
    }
    log {
    	verbose = 0
    	silent = 0
    	syslog = 1
    	overwrite = 0
    	level = 0
    	indent = 1
    	command_names = 0
    	prefix = "  "
    	activation = 0
    	debug_classes = [ "memory", "devices", "activation", "allocation", "lvmetad", "metadata", "cache", "locking", "lvmpolld", "dbus" ]
    }
    backup {
    	backup = 1
    	backup_dir = "/etc/lvm/backup"
    	archive = 1
    	archive_dir = "/etc/lvm/archive"
    	retain_min = 10
    	retain_days = 30
    }
    shell {
    	history_size = 100
    }
    global {
    	umask = 077
    	test = 0
    	units = "h"
    	si_unit_consistency = 1
    	suffix = 1
    	activation = 1
    	proc = "/proc"
    	etc = "/etc"
    	locking_type = 1
    	wait_for_locks = 1
    	fallback_to_clustered_locking = 1
    	fallback_to_local_locking = 1
    	locking_dir = "/run/lock/lvm"
    	prioritise_write_locks = 1
    	abort_on_internal_errors = 0
    	detect_internal_vg_cache_corruption = 0
    	metadata_read_only = 0
    	mirror_segtype_default = "raid1"
    	raid10_segtype_default = "raid10"
    	sparse_segtype_default = "thin"
    	use_lvmetad = 1
    	use_lvmlockd = 0
    	system_id_source = "none"
    	use_lvmpolld = 1
    	notify_dbus = 1
    }
    activation {
    	checks = 0
    	udev_sync = 1
    	udev_rules = 1
    	verify_udev_operations = 0
    	retry_deactivation = 1
    	missing_stripe_filler = "error"
    	use_linear_target = 1
    	reserved_stack = 64
    	reserved_memory = 8192
    	process_priority = -18
    	raid_region_size = 512
    	readahead = "auto"
    	raid_fault_policy = "warn"
    	mirror_image_fault_policy = "remove"
    	mirror_log_fault_policy = "allocate"
    	snapshot_autoextend_threshold = 100
    	snapshot_autoextend_percent = 20
    	thin_pool_autoextend_threshold = 100
    	thin_pool_autoextend_percent = 20
    	use_mlockall = 0
    	monitoring = 1
    	polling_interval = 15
    	activation_mode = "degraded"
    }
    dmeventd {
    	mirror_library = "libdevmapper-event-lvm2mirror.so"
    	snapshot_library = "libdevmapper-event-lvm2snapshot.so"
    	thin_library = "libdevmapper-event-lvm2thin.so"
    }
    
    5) reboot to rescue system
    lvremove /dev/vgsys/snap 
    reboot
    
    ... and the machine boots in normal way ...
    
    txt file icon CentOS_LVM_ERROR.txt (16,266 bytes) 2017-03-27 18:11 +

-Relationships
+Relationships

-Notes

~0028923

kabe (reporter)

I succeeded in reproducing something similar.

- Install CentOS 7 with default disk allocation.
  Default Volume Group: "cl"
  Root Logical Volume: "cl/root"
  Swap Logical Volume: "cl/swap"

- After reboot, erase the swap to free up the Volume Group.
  # swapoff /dev/mapper/swap
  # lvremove cl/swap

- Delete /dev/mapper/cl-swap entry from /etc/fstab.

- Make a non-thin snapshot of root partition (this seems optional)
  # lvcreate -s cl/root -n root_snap -L 500m

- Reboot.

The initramfs will stop after
    File System Check on /dev/mapper/cl-root
(Actually, it is waiting for /bin/dracut-initqueue to finish)

After 120 seconds ((rd.retry=180)/3*2),
    dracut-initqueue[257]: Warning: dracut-initqueue timeout - starting timeout scripts
will be repeated in (sleep 0.5) interval.

After 180 seconds, my installation resumed normal booting.


There was a kernel commandline option remaining:
    rd.lvm.lv=cl/swap
so, during boot, dracut-initqueue seems to be waiting for /dev/cl/swap to appear.

But you deleted the swap, so it never appears.
So, dracut-initqueue tries to wait for 180 seconds and gives up.

Deleting "rd.lvm.lv=cl/swap" option from /boot/grub2/grub.cfg
fixed my installation.
(Yours may have "rd.lvm.lv=vgsys/swap")
Boots flulently, even with root partition snapshot existing.


Your installation dropped to rdshell, mine resumed normal booting.
I don't know how to exactly reproduce the symptom described in
https://www.centos.org/forums/viewtopic.php?f=47&t=59006

The timeout could be adjusted by "rd.retry=180" kernel option.


* What do you have on kernel boot option in /boot/grub2/grub.cfg?
* What is current /etc/fstab content?

~0028927

kabe (reporter)

Considering date of this bugreport had been originally submitted,
I tried CentOS 7.2 .
I couldn't still reproduce the phenomenon which root filesystem gets lost when
root snapshot exists.
Using root=UUID=..., which is same on root and snapshot,
doesn't make dracut confused, and boots properly.

Although, when given bogus "rd.lvm.lv=centos/swap" kernel option,
dracut of CentOS 7.2 doesn't continue after timeout and
always drops into rdshell.
Just "exit" the rdshell and the boot continues.
Came close, but not quite.

~0028934

sz1305 (reporter)

I tried CentOS 7.3 with all patches (2017-03-25). I could still reproduce the error.
I will attatch two files, a screenlog with some configurations (CentOS version fstab, grub.cfg etc.) and a screenshot from error message.

~0028935

sz1305 (reporter)

there are a screenshot (error message)

~0028936

kabe (reporter)

Thank you for the info dump.

Your grub.cfg looks healthy.
I cannot derive a reason why you cannot boot with a snapshot...
It seems very odd that you could boot rescue menu without issues.


Screenshot>> [*** ] A start job is running for dev-mapper-vgsys\x2droot.device
This looks odd... I think it should be "dev-vgsys-root.device".
Since you updated to CentOS 7.3, initqueue is continuing after timeout and systemd is stuck.
Anyway I want to reproduce this but couldn't.

>> lvcreate --snapshot --size 200m --name root_2017-03-25_11-00 /dev/root/vgsys/root
This a typo, right? it emits error.
You usually will say
|| lvcreate --snapshot --size 200m --name root_2017-03-25_11-00 vgsys/root
|| lvcreate --snapshot --size 200m --name root_2017-03-25_11-00 /dev/vgsys/root


Q. Are you creating snapshot by the given command line, or using some automated tool?
Q. Is your system's "systemd" and "dracut" package updated?
Q. Does "rpm -qV systemd dracut kernel" report any weird errors?
Q. Does removing "LANG=de_DE.UTF-8" from kernel option change things?
Q. Does using snapshot name withOUT hyphen (use snap_root_YYYY_mm_dd) change things?
Q. How did you partition the machine? VG name "vgsys" isn't a default.
Q. Is physical partition MBR or GPT type?


* Things I want you to try:
It's better to have serial console attached, to capture console output.
(screen capture isn't nice for pasting, you know)
Serials consoles are also available in virtualized environment.

You would like to add "rd.retry=20" to reduce waiting for 180 secs to 20 secs.
Also, delete "rhgb quiet" to emit more transient messages.

After creating a snapshot,
reboot with "rd.break=initqueue" kernel option, and drop to emergency shell.
I want to see the outputs of the following commands.

initqueue:/# ls -lR /usr/lib/dracut/hooks/initqueue/
initqueue:/# sh -x /sbin/lvm_scan
initqueue:/# lvm lvs
initqueue:/# lvm vgdisplay
initqueue:/# lvm pvdisplay
initqueue:/# ls -lR /dev/disk
initqueue:/# ls -lR /dev/mapper
initqueue:/# cat /proc/modules
initqueue:/# sed -e '/^[\t ]*#/d; /^$/d' /etc/lvm/lvm.conf ;#contents of /etc/lvm/lvm.conf
You can scrollback half a screen by Shift+PageUp.

initqueue:/# lvm lvremove vgsys/snap_whatever ;#try delete snapshot, if it's visible
initqueue:/# lvm_scan
initqueue:/# lvm lvs
initqueue:/# exit ;# continue boot

And, boot the rescue menu (the one you can boot with)
with "rd.break=initqueue" kernel option:
initqueue:/# uname -a ;# should show an old kernel version
initqueue:/# cat /proc/modules


I could reproduce something similar by
"lvchange -ky /dev/centos/root", but I bet this is not the case;
it will make ALL grub menus to fail boot.

~0028937

kabe (reporter)

I found a very identical RHBZ:
Creating an LVM snapshot can render the system unbootable
https://bugzilla.redhat.com/show_bug.cgi?id=1287940

The errata seems to say updating to latest dracut package fixes this.
Maybe re-installing the kernel (initrd regenerate) changes something.

~0028945

sz1305 (reporter)

Hello, thanks for your informations. I tested your instructions (see atached file)

When I create a new initramfs "dracut -vf" after I cretated a snapshot with a name with a simple(!) name "lvcreate --snapshot --name snap --size 200m /dev/vgsys/root", the machine boots properly. But, create a snapshot, reboot in rescue mode and generate a new initramfs don't works properly.

thanks ... and sorry for my English ;)

~0028946

kabe (reporter)

Thanks a lot for the detailed log. It really helps.

>> initqueue:/# sh -x /sbin/lvm_scan
...
>> + lvm lvchange --yes -ay --sysinit vgsys/root
>> + vinfo
>> + read line
>> + info modprobe: FATAL: Module dm-snapshot not found.
...

Aha! You're hitting the https://bugzilla.redhat.com/show_bug.cgi?id=1287940 .

I also succeeded in reproducing this;
If you update CentOS 7.0 to latest kernel and not dracut,
the new kernel couldn't boot when snapshot is existing.


To fix this, you must
1. update the dracut package to latest version, then
2. (re-)install the kernel
   (is the easiest way to rebuid the initrd with the updated dracut).
Don't do this in single yum session;
it could break resulting in initramfs not including the dm-snapshot.ko module.

To confirm its correct, the new initramfs should contain dm-snapshot.ko:
# lsinitrd /boot/initramfs-3.10.0-514.el7.x86_64.img | grep dm-snapshot.ko
-rw-r--r-- 1 root root 70893 Nov 23 02:29 usr/lib/modules/3.10.0-514.el7.x86_64/kernel/drivers/md/dm-snapshot.ko

The failing initramfs doesn't and booting initramfs does include dm-snapshot.ko.


The name of the snapshot should not affect this bug.
root_snap_2017-01-02_03-04 should work if updated correctly.

If you succeeded in fixing the problem, please post a reply
to request to close this bugreport as "resolved".

~0028955

sz1305 (reporter)

# lsinitrd lsinitrd /boot/initramfs-$(uname -r).img | grep dm-snapshot.ko
no kernel module found
# lvcreate --snapshot --name root_2017-03-28_18-12 --size 200m /dev/vgsys/root
# dracut -fv
# lsinitrd lsinitrd /boot/initramfs-$(uname -r).img | grep dm-snapshot.ko
kernel module found
# reboot
machine reboots propperly, well done and THANKS!

~0028956

sz1305 (reporter)

resolved, please clode this bugreport.
+Notes

-Issue History
Date Modified Username Field Change
2016-12-18 14:44 sz1305 New Issue
2017-03-23 07:43 kabe Note Added: 0028923
2017-03-24 06:10 kabe Note Added: 0028927
2017-03-25 10:59 sz1305 File Added: screenlog.txt
2017-03-25 10:59 sz1305 Note Added: 0028934
2017-03-25 11:18 sz1305 File Added: Screenshot.png
2017-03-25 11:18 sz1305 Note Added: 0028935
2017-03-27 05:46 kabe Note Added: 0028936
2017-03-27 07:51 kabe Note Added: 0028937
2017-03-27 18:11 sz1305 File Added: CentOS_LVM_ERROR.txt
2017-03-27 18:11 sz1305 Note Added: 0028945
2017-03-28 02:04 kabe Note Added: 0028946
2017-03-28 17:04 sz1305 Note Added: 0028955
2017-03-28 17:06 sz1305 Note Added: 0028956
+Issue History