View Issue Details

Jump to NotesJump to History
IDProjectCategoryView StatusDate SubmittedLast Update
0018118CentOS-7kernelpublic2021-03-23 00:332021-03-25 01:38
Reporterzhaozhen Assigned To 
PriorityhighSeveritycrashReproducibilityalways
Status newResolutionopen 
Product Version7.6.1810 
Summary0018118: xfs: system crash caused by null bp->b_pages
DescriptionWe have encountered the following problems several times:
    1、Hardware problem causes block device loss.
    2、Continue to send IO requests to the block device.
    3、The system possibly crash after a few hours.
Steps To ReproduceWe have encountered the following problems several times:
    1、Hardware problem causes block device loss.
    2、Continue to send IO requests to the block device.
    3、The system possibly crash after a few hours.
Additional InformationOOPs:

[98469.932030] RSP: 0018:ffff9025dd727c38 EFLAGS: 00010256
[98469.932523] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 000000000000000b
[98469.933040] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff9005c51fb300
[98469.933566] RBP: ffff9025dd727c68 R08: ffff9005c51fb370 R09: ffff9025d4fee180
[98469.934103] R10: ffff9035ff8df120 R11: ffffcbcd0169d000 R12: ffff9005d9b90000
[98469.934646] R13: 0000000000000020 R14: ffff9005c51fb300 R15: 0000000000000000
[98469.935201] FS: 00007fadceb91700(0000) GS:ffff9035ff8c0000(0000) knlGS:0000000000000000
[98469.935772] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[98469.936357] CR2: 0000000000000000 CR3: 0000002332364000 CR4: 00000000003407e0
[98469.936956] Call Trace:
[98469.937581] [<ffffffffc0810e68>] ? xfs_inode_buf_verify+0x98/0x100 [xfs]
[98469.938233] [<ffffffffc081d2a8>] ? xfs_buf_delwri_submit_buffers+0x128/0x230 [xfs]
[98469.938901] [<ffffffffc0810ee0>] xfs_inode_buf_write_verify+0x10/0x20 [xfs]
[98469.939580] [<ffffffffc081b1b7>] _xfs_buf_ioapply+0x97/0x460 [xfs]
[98469.940258] [<ffffffff97ab72a4>] ? wake_up_worker+0x24/0x30
[98469.940934] [<ffffffff97ab7b12>] ? insert_work+0x62/0xa0
[98469.941620] [<ffffffff97ad6830>] ? wake_up_state+0x20/0x20
[98469.942326] [<ffffffffc081d2a8>] ? xfs_buf_delwri_submit_buffers+0x128/0x230 [xfs]
[98469.943059] [<ffffffffc081cfcc>] +0x6c/0x220 [xfs]
[98469.943794] [<ffffffffc081d2a8>] xfs_buf_delwri_submit_buffers+0x128/0x230 [xfs]
[98469.944549] [<ffffffffc081e0c0>] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[98469.945321] [<ffffffffc084cc60>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[98469.946098] [<ffffffffc081e0c0>] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[98469.946889] [<ffffffffc084cebf>] xfsaild+0x25f/0x6f0 [xfs]
[98469.947692] [<ffffffffc084cc60>] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[98469.948655] [<ffffffff97ac1cb1>] kthread+0xd1/0xe0
[98469.949520] [<ffffffff97ac1be0>] ? insert_kthread_work+0x40/0x40
[98469.950350] [<ffffffff98174c24>] ret_from_fork_nospec_begin+0xe/0x21
[98469.951208] [<ffffffff97ac1be0>] ? insert_kthread_work+0x40/0x40
[98469.952056] Code: 48 89 e5 48 85 c0 74 0b 48 01 f0 5d c3 66 0f 1f 44 00


bp information:

crash> struct xfs_buf 0xffff9005c51fb300
struct xfs_buf {
  b_rhash_head = {
    next = 0x0
  },
  b_bn = 481790464,
  b_length = 128,
  b_hold = {
    counter = 2
  },
  b_lru_ref = {
    counter = 0
  },
  b_flags = 1048626,
  b_sema = {
    lock = {
      raw_lock = {
        val = {
          counter = 0
        }
      }
    },
    count = 0,
    wait_list = {
      next = 0xffff9005c51fb328,
      prev = 0xffff9005c51fb328
    }
  },
  b_lru = {
    next = 0xffff9005c51fb338,
    prev = 0xffff9005c51fb338
  },
  b_lock = {
    {
      rlock = {
        raw_lock = {
          val = {
            counter = 0
          }
        }
      }
    }
  },
  b_state = 3,
  b_io_error = 0,
  b_waiters = {
    lock = {
      {
        rlock = {
          raw_lock = {
            val = {
              counter = 0
            }
          }
        }
      }
    },
    task_list = {
      next = 0xffff9005c51fb360,
      prev = 0xffff9005c51fb360
    }
  },
  b_list = {
    next = 0xffff9005c51fb370,
    prev = 0xffff9005c51fb370
  },
  b_pag = 0xffff9005de557400,
  b_target = 0xffff9005ddff0d80,
  b_addr = 0x0,
  b_ioend_work = {
    data = {
      counter = 1920
    },
    entry = {
      next = 0xffff9005c51fb3a0,
      prev = 0xffff9005c51fb3a0
    },
    func = 0xffffffffc081ce80 <xfs_buf_ioend_work>
  },
  b_ioend_wq = 0xffff9035d433bc00,
  b_iodone = 0xffffffffc0843220 <xfs_buf_iodone_callbacks>,
  b_iowait = {
    done = 0,
    wait = {
      lock = {
        {
          rlock = {
            raw_lock = {
              val = {
                counter = 0
              }
            }
          }
        }
      },
      task_list = {
        next = 0xffff9005c51fb3d8,
        prev = 0xffff9005c51fb3d8
      }
    }
  },
  b_fspriv = 0xffff9005c5b9c690,
  b_transp = 0x0,
  b_pages = 0x0,
  b_page_array = {0x0, 0x0},
  b_maps = 0xffff9005c51fb418,
  __b_map = {
    bm_bn = 481790464,
    bm_len = 128
  },
  b_map_count = 1,
  b_io_length = 128,
  b_pin_count = {
    counter = 0
  },
  b_io_remaining = {
    counter = 1
  },
  b_page_count = 16,ƒb
  b_offset = 0,
  b_error = 0,
  b_retries = 0,
  b_first_retry_time = 0,
  b_last_error = -5,
  b_ops = 0xffffffffc085cf00 <xfs_inode_buf_ops>
TagsNo tags attached.
abrt_hash
URL

Activities

zhaozhen

zhaozhen

2021-03-23 00:39

reporter   ~0038316

From vmcore, wee found that b_pages=NULL but b_page_count=16.

file: xfs_log.c function: xlog_sync, it seems current code not handle allocate memory fail.

if (split) {
                bp = iclog->ic_log->l_xbuf;
                XFS_BUF_SET_ADDR(bp, 0); /* logical 0 */
                xfs_buf_associate_memory(bp,
                                (char *)&iclog->ic_header + count, split);
                bp->b_fspriv = iclog;
                bp->b_flags &= ~XBF_FLUSH;
                bp->b_flags |= (XBF_ASYNC | XBF_SYNCIO | XBF_WRITE | XBF_FUA);

                ASSERT(XFS_BUF_ADDR(bp) <= log->l_logBBsize-1);
                ASSERT(XFS_BUF_ADDR(bp) + BTOBB(count) <= log->l_logBBsize);

                /* account for internal log which doesn't start at block #0 */
                XFS_BUF_SET_ADDR(bp, XFS_BUF_ADDR(bp) + log->l_logBBstart);
                error = xlog_bdstrat(bp);
                if (error) {
                        xfs_buf_ioerror_alert(bp, "xlog_sync (split)");
                        return error;
                }
        }
gadedat489

gadedat489

2021-03-24 07:28

reporter   ~0038320

this was fixed in 5.9.11 with https://www.spinics.net/lists/linux-xfs/msg46873.html

My website: https://trangtriqc.blogspot.com
zhaozhen

zhaozhen

2021-03-25 01:37

reporter   ~0038328

I think, it is not the same issue. this issue caused by null b_pages.

15205901.386974] RIP: 0010:xfs_buf_offset+0x32/0x60 [xfs]
[15205901.388044] RSP: 0018:ffffb28ba9b3bc68 EFLAGS: 00010246
[15205901.389021] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 000000000000000b
[15205901.390016] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffff88627bebf000
[15205901.391075] RBP: ffffb28ba9b3bc98 R08: ffff88627bebf000 R09: 00000001802a000d
[15205901.392031] R10: ffff88521f3a0240 R11: ffff88627bebf000 R12: ffff88521041e000
[15205901.392950] R13: 0000000000000020 R14: ffff88627bebf000 R15: 0000000000000000
[15205901.393858] FS: 0000000000000000(0000) GS:ffff88521f380000(0000) knlGS:0000000000000000
[15205901.394774] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[15205901.395756] CR2: 0000000000000000 CR3: 000000099bc09001 CR4: 00000000007606e0
[15205901.396904] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[15205901.397869] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[15205901.398836] PKRU: 55555554
[15205901.400111] Call Trace:
[15205901.401058] ? xfs_inode_buf_verify+0x8e/0xf0 [xfs]
[15205901.402069] ? xfs_buf_delwri_submit_buffers+0x16d/0x2b0 [xfs]
[15205901.403060] xfs_inode_buf_write_verify+0x10/0x20 [xfs]
[15205901.404017] _xfs_buf_ioapply+0x88/0x410 [xfs]
[15205901.404990] ? xfs_buf_delwri_submit_buffers+0x16d/0x2b0 [xfs]
[15205901.405929] xfs_buf_submit+0x63/0x200 [xfs]
[15205901.406801] xfs_buf_delwri_submit_buffers+0x16d/0x2b0 [xfs]
[15205901.407675] ? xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[15205901.408540] ? xfs_inode_item_push+0xb7/0x190 [xfs]
[15205901.409395] xfs_buf_delwri_submit_nowait+0x10/0x20 [xfs]
[15205901.410249] xfsaild+0x29a/0x780 [xfs]
[15205901.411121] kthread+0x109/0x140
[15205901.411981] ? xfs_trans_ail_cursor_first+0x90/0x90 [xfs]
[15205901.412785] ? kthread_park+0x60/0x60
[15205901.413578] ret_from_fork+0x2a/0x40
zhaozhen

zhaozhen

2021-03-25 01:38

reporter   ~0038329

From vmcore, b_pages = 0x0


b_fspriv = 0xffff9005c5b9c690,
  b_transp = 0x0,
  b_pages = 0x0,
  b_page_array = {0x0, 0x0},
  b_maps = 0xffff9005c51fb418,
  __b_map = {
    bm_bn = 481790464,
    bm_len = 128
  },
  b_map_count = 1,
  b_io_length = 128,

Issue History

Date Modified Username Field Change
2021-03-23 00:33 zhaozhen New Issue
2021-03-23 00:39 zhaozhen Note Added: 0038316
2021-03-24 07:28 gadedat489 Note Added: 0038320
2021-03-25 01:37 zhaozhen Note Added: 0038328
2021-03-25 01:38 zhaozhen Note Added: 0038329