Dell Precision Tower 5820 - FlexBay MiniSAS PCIe NVMe SSD not recognized

Sebastian Roth

@george1421 Found this in the diff… no idea if that could be related:

diff -Nur linux-4.13/drivers/nvme/host/pci.c fedora-kernel-4.13.9-300.fc27/drivers/nvme/host/pci.c
--- linux-4.13/drivers/nvme/host/pci.c  2017-09-03 22:56:17.000000000 +0200
+++ fedora-kernel-4.13.9-300.fc27/drivers/nvme/host/pci.c       2017-10-23 22:25:50.000000000 +0200
@@ -93,7 +93,7 @@
        struct mutex shutdown_lock;
        bool subsystem;
        void __iomem *cmb;
-       dma_addr_t cmb_dma_addr;
+       pci_bus_addr_t cmb_bus_addr;
        u64 cmb_size;
        u32 cmbsz;
        u32 cmbloc;
@@ -1218,7 +1218,7 @@
        if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
                unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
                                                      dev->ctrl.page_size);
-               nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
+               nvmeq->sq_dma_addr = dev->cmb_bus_addr + offset;
                nvmeq->sq_cmds_io = dev->cmb + offset;
        } else {
                nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
@@ -1517,7 +1517,7 @@
        resource_size_t bar_size;
        struct pci_dev *pdev = to_pci_dev(dev->dev);
        void __iomem *cmb;
-       dma_addr_t dma_addr;
+       int bar;
 
        dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
        if (!(NVME_CMB_SZ(dev->cmbsz)))
@@ -1530,7 +1530,8 @@
        szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
        size = szu * NVME_CMB_SZ(dev->cmbsz);
        offset = szu * NVME_CMB_OFST(dev->cmbloc);
-       bar_size = pci_resource_len(pdev, NVME_CMB_BIR(dev->cmbloc));
+       bar = NVME_CMB_BIR(dev->cmbloc);
+       bar_size = pci_resource_len(pdev, bar);
 
        if (offset > bar_size)
                return NULL;
@@ -1543,12 +1544,11 @@
        if (size > bar_size - offset)
                size = bar_size - offset;
 
-       dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(dev->cmbloc)) + offset;
-       cmb = ioremap_wc(dma_addr, size);
+       cmb = ioremap_wc(pci_resource_start(pdev, bar) + offset, size);
        if (!cmb)
                return NULL;
 
-       dev->cmb_dma_addr = dma_addr;
+       dev->cmb_bus_addr = pci_bus_address(pdev, bar) + offset;
        dev->cmb_size = size;
        return cmb;
 }
@@ -1609,18 +1609,16 @@
        dev->host_mem_descs = NULL;
 }
 
-static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
+static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
+               u32 chunk_size)
 {
        struct nvme_host_mem_buf_desc *descs;
-       u32 chunk_size, max_entries, len;
+       u32 max_entries, len;
        dma_addr_t descs_dma;
        int i = 0;
        void **bufs;
        u64 size = 0, tmp;
-       /* start big and work our way down */
-       chunk_size = min(preferred, (u64)PAGE_SIZE << MAX_ORDER);
-retry:
        tmp = (preferred + chunk_size - 1);
        do_div(tmp, chunk_size);
        max_entries = tmp;
@@ -1647,15 +1645,9 @@
                i++;
        }
 
-       if (!size || (min && size < min)) {
-               dev_warn(dev->ctrl.device,
-                       "failed to allocate host memory buffer.\n");
+       if (!size)
                goto out_free_bufs;
-       }
 
-       dev_info(dev->ctrl.device,
-               "allocated %lld MiB host memory buffer.\n",
-               size >> ilog2(SZ_1M));
        dev->nr_host_mem_descs = i;
        dev->host_mem_size = size;
        dev->host_mem_descs = descs;
@@ -1676,21 +1668,35 @@
        dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
                        descs_dma);
 out:
-       /* try a smaller chunk size if we failed early */
-       if (chunk_size >= PAGE_SIZE * 2 && (i == 0 || size < min)) {
-               chunk_size /= 2;
-               goto retry;
-       }
        dev->host_mem_descs = NULL;
        return -ENOMEM;
 }
 
-static void nvme_setup_host_mem(struct nvme_dev *dev)
+static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
+{
+       u32 chunk_size;
+
+       /* start big and work our way down */
+       for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
+            chunk_size >= PAGE_SIZE * 2;
+            chunk_size /= 2) {
+               if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
+                       if (!min || dev->host_mem_size >= min)
+                               return 0;
+                       nvme_free_host_mem(dev);
+               }
+       }
+
+       return -ENOMEM;
+}
+
+static int nvme_setup_host_mem(struct nvme_dev *dev)
 {
        u64 max = (u64)max_host_mem_size_mb * SZ_1M;
        u64 preferred = (u64)dev->ctrl.hmpre * 4096;
        u64 min = (u64)dev->ctrl.hmmin * 4096;
        u32 enable_bits = NVME_HOST_MEM_ENABLE;
+       int ret = 0;
 
        preferred = min(preferred, max);
        if (min > max) {
@@ -1698,7 +1704,7 @@
                        "min host memory (%lld MiB) above limit (%d MiB).\n",
                        min >> ilog2(SZ_1M), max_host_mem_size_mb);
                nvme_free_host_mem(dev);
-               return;
+               return 0;
        }
 
        /*
@@ -1712,12 +1718,21 @@
        }
 
        if (!dev->host_mem_descs) {
-               if (nvme_alloc_host_mem(dev, min, preferred))
-                       return;
+               if (nvme_alloc_host_mem(dev, min, preferred)) {
+                       dev_warn(dev->ctrl.device,
+                               "failed to allocate host memory buffer.\n");
+                       return 0; /* controller must work without HMB */
+               }
+
+               dev_info(dev->ctrl.device,
+                       "allocated %lld MiB host memory buffer.\n",
+                       dev->host_mem_size >> ilog2(SZ_1M));
        }
 
-       if (nvme_set_host_mem(dev, enable_bits))
+       ret = nvme_set_host_mem(dev, enable_bits);
+       if (ret)
                nvme_free_host_mem(dev);
+       return ret;
 }
 
 static int nvme_setup_io_queues(struct nvme_dev *dev)
@@ -2161,8 +2176,11 @@
                                 "unable to allocate dma for dbbuf\n");
        }
 
-       if (dev->ctrl.hmpre)
-               nvme_setup_host_mem(dev);
+       if (dev->ctrl.hmpre) {
+               result = nvme_setup_host_mem(dev);
+               if (result < 0)
+                       goto out;
+       }
 
        result = nvme_setup_io_queues(dev);
        if (result)

george1421

@sebastian-roth Just to add in a bit more, looking at the relevant dmsg lines this is what I see

[    0.402292] pci 0000:b3:00.0: [1c5c:1527] type 00 class 0x010802
[    0.402308] pci 0000:b3:00.0: reg 0x10: [mem 0xfb500000-0xfb503fff 64bit]
[    0.503618] nvme nvme0: pci function 0000:b3:00.0
[    0.716693]  nvme0n1: p1 p2 p3 p4
[    3.638397] nvme nvme0: failed to set APST feature (-19)

If I understand it correctly the kernel is seeing the nvme disk because it knows its name of nvme0n1 and it see 4 partitions on the disk.

I wonder if the patch you found made it into the main stream code? Since FC27 is using 4.13.9 and we’re testing 4.17.13

Tom Elliott

I’m hoping you guys get this figured out. Sorry been distant for the last couple weeks. Just started a new position so not a lot of time right now during the day.

Of note, Linux kernel is now in the 4.18 series. I’m going to be building a new kernel relatively shortly, just need time (probably this weekend.)

@george1421 Would you mind building a plain jane 4.18 based on the TomElliott.config? Don’t forget to do the patches:

https://wiki.fogproject.org/wiki/index.php?title=Build_TomElliott_Kernel#Additional_Patches

In particular the patches for:
mmc (The path used to be mmc/card/ is now mmc/core/) and scsi/storvsc (this is whats supposed to help with the mbr/gpt erasing issue).

I believe the e1000 patch was already fixed in 4.17 so we shouldn’t need to do anything, but you can open the and check.

Sorry I can’t be more useful right now.

Either way, awesome work guys.

george1421

@tom-elliott said in Dell Precision Tower 5820 - FlexBay MiniSAS PCIe NVMe SSD not recognized:

Would you mind building a plain jane 4.18 based on the TomElliott.config? Don’t forget to do the patches:

Do you want 4.18 based on FOG standard configuration, or where I’m at with 4.17.13?

Either way, I probably should apply the additional patches to 4.17.13. Just as a side note, I didn’t see the patch for the slow GPT disk issue you mentioned a while ago. Do you have that info too?

Tom Elliott

@george1421 On the wiki page, that’s the storvsc portion of the additional patches.

For understanding,

If the page size is 4096, it’s supposed to do stuff. However, not all disks use a 4096 page size. This is noticed particularly when initializing/erasing mbr/gpt structures on a disk. It’s just strange, to me, that it only seems to be impacting Windows based installs. At least from what I’ve seen.

george1421

@sebastian-roth The kernel patch and diffs you found for 4.13 have been integrated into the mainstream code. I confirmed they were there in 4.17.13

I’m going to rebuild with that config parameter you mentioned form that Arch post. I also integrated the patches from the FOG Wiki site. For now I think I’m going to stick with 4.17.13 for developing the kernel, I have a rule to never install a XX.0 release of anything (thank you Bill Gates).

george1421

I did recompile the 4.17.13 kernel with the powersave setting from the Arch article, plus FOG kernel patches and then added support for Microsoft Surface network adapter (patch info found here: https://forums.fogproject.org/topic/10943/surface-pro-4-registration-issues ). This kernel is ‘M’ release. Please test it out.

hlalex

@george1421 version M logs:

hlalex

@george1421 FlexBay Parts:

414-BBBV : PCIe SSD (Front PCIe FlexBay)
Part Number Quantity Description
5G90D 1 INFORMATION, FRONT, PERIPHERAL COMPONENT INTERCONNECT EXPRESS , SOFTWARE SUPPORT DISKETTE, BOOT

401-ABJT : M.2 512GB PCIe NVMe Class 40 S olid State Drive
Part Number Quantity Description
XMW6J 1 SSDR, 512G, P34, 80S3, HYNIX, PC401

400-AVDR : Dell M.2 carrier
Part Number Quantity Description
66XHV 1 ASSEMBLY, DRIVE, BAY (DRIVE BAY), M.2, MODULE

hlalex

@george1421 Ok, got some data from windows device manager for the FlexBay drive:

PCI Memory Controller [ this was an uninitialized device, figured it best to include as it is a PCI device but probably not relevant ]

PCI\VEN_8086&DEV_A2A1&SUBSYS_07381028&REV_00

Everything below l pertains to the M.2 connected to the FlexBay
Disk drive
Device instance path

SCSI\DISK&VEN_NVME&PROD_PC401_NVME_SK_HY\5&1A7BC20F&0&000000

HW IDs

SCSI\DiskNVMe____PC401_NVMe_SK_hy3E00
SCSI\DiskNVMe____PC401_NVMe_SK_hy
SCSI\DiskNVMe____
SCSI\NVMe____PC401_NVMe_SK_hy3
NVMe____PC401_NVMe_SK_hy3
GenDisk

Status

0180200A
DN_DRIVER_LOADED
DN_STARTED
DN_DISABLEABLE
DN_NT_ENUMERATOR
DN_NT_DRIVER

Class Guid

{4d36e967-e325-11ce-bfc1-08002be10318}

Device stack

\Driver\partmgr
\Driver\Disk
\Driver\EhStorClass
\Driver\stornvme

Driver node strong name

disk.inf:6d166ee9677c725c:disk_install.NT:10.0.16299.371:GenDisk

hlalex

@george1421 version N:

hlalex

hlalex

@george1421 Success!!

Here are the logs:

One thing I noticed is I got some weird DHCP messages before it booted to the debug console. I will try to replicate and grab an image if possible.

Edit: Shot of the DHCP messages. It says it didn’t get an IP, but it actually does. After pressing [enter] it drops to the debug shell.

Sebastian Roth

@george1421 I am really keen to hear what you did with kernel -300m as this seems to make a difference… right track I suppose. Maybe this is some kind of timeout?!

george1421

@sebastian-roth Sorry its been a long day of poking and pulling the linux kernel.
Today I tried the setting from the Arch document with no luck in the ‘M’ release.
Then I tried just removing all of the power saving code out of the kernel as well as removing all acpi code, with no success. The ‘N’ release.
Next I was about to give up then I got the idea since the FC27 kernel worked when FC27 live was booted I decided to take the tar file you linked below and just straight copy over the ‘M’ config file and then built the kernel. I did this to test 2 ideas. 1) Did I have the right kernel options selected from the 4.17.13 release 2) Was the FC devs able to patch the 4.13.9 kernel to make it work with the OPs hardware. Both tests were successful. That is what @hlalex posted in the bzImage4139-300m.log log file. In another thread an OP had an issue with the current FOS kernel and an Microsoft usb network adapter. I added the patch into 4.17.13 M kernel build. That network driver was then discovered by FOS, but FOS still has the delayed creation of the GPT partition. So that mystery was not solved as of now either. I was kind of hoping the updated FOS kernel would have addressed that but not as of now.

Back on point so… It would be ideal to understand what the FC kernel dev guys did to linux 4.13.9 to create 4.13.9-300. I assume the -300 means there was 300 patches to the stock 4.13.9 kernel (??). The only thing I can think is to do a (mega)diff between linux 4.13.9 and the FC 4.13.9-300. To see what has changed.

george1421

@sebastian-roth It appears the -100 -200 -300 numbering is based on the FC release and not to indicate a patch level.

So that means I’m back to trying to figure out how to diff all of the files between FC version and linux version of 4.13.9.

george1421

Will you give the linux 4.13.9 kernel a shot before you send the hardware back this week? This last test will tell use if the fix is in the 4.13.9 kernel or something that FC did with 4.13.9-300 kernel.

hlalex

@george1421 Sure thing, I’m keeping them around as long as possible so we can get as much info as possible from them.

hlalex

@george1421 I have to get these drives shipped out by 3:30pm EST today. I anyone has any additional test let me know and I will get anything I can.

Otherwise, Thanks for all the help!

george1421

@hlalex Thank you for your help. Go ahead and send them back at this time. It looks like kernel changes between 4.13.x and 4.17.x has disabled these types of drives. My only option would be to see if 4.18 has fixed the issue. But at this point you have a solution and the hardware must go back. So we’ll park testing until someone else has this issue. Again thank you for all of your testing and data collection. We would not have made it this far without your help.

Dell Precision Tower 5820 - FlexBay MiniSAS PCIe NVMe SSD not recognized

157

11.6k

17.1k

154.6k