nvme hardware queue 初始化流程

主要步骤

To setup and initialize I/O Submission Queues and I/O Completion Queues for use, host software follows these steps:

1. Configures the Admin Submission and Completion Queues by initializing the Admin Queue Attributes (AQA), Admin Submission Queue Base Address (ASQ), and Admin Completion Queue Base Address (ACQ) registers appropriately;
2. Submits a Set Features command with the Number of Queues attribute to request the desired number of I/O Submission Queues and I/O Completion Queues. The completion queue entry for this Set Features command indicates the number of I/O Submission Queues and I/O Completion Queues allocated by the controller;
3. Determines the maximum number of entries supported per queue (CAP.MQES) and whether the queues are required to be physically contiguous (CAP.CQR);
4. Creates the desired I/O Completion Queues within the limitations of the number allocated by the controller and the queue attributes supported (maximum entries and physically contiguous requirements) by using the Create I/O Completion Queue command; and
5. Creates the desired I/O Submission Queues within the limitations of the number allocated by the controller and the queue attributes supported (maximum entries and physically contiguous requirements) by using the Create I/O Submission Queue command.

At the setup and initialized and may be used to complete I/O commands.

参考代码

spdk/examples/nvme/perf/perf.c

static void
attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
          struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
{
        struct trid_entry       *trid_entry = cb_ctx;
        struct spdk_pci_addr    pci_addr;
        struct spdk_pci_device  *pci_dev;
        struct spdk_pci_id      pci_id;

        g_controllers_found++;
        if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) {
                printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n",
                       trid->traddr, trid->trsvcid,
                       trid->subnqn);
        } else {
                if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) {
                        return;
                }

                pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr);
                if (!pci_dev) {
                        return;
                }

                pci_id = spdk_pci_device_get_id(pci_dev);

                printf("Attached to NVMe Controller at %s [%04x:%04x]\n",
                       trid->traddr,
                       pci_id.vendor_id, pci_id.device_id);
        }

        register_ctrlr(ctrlr, trid_entry);
}

static int
register_controllers(void)
{
        struct trid_entry *trid_entry;

        printf("Initializing NVMe Controllers\n");

        TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) {
                if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) {
                        fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n",
                                trid_entry->trid.traddr);
                        return -1;
                }
        }

        return 0;
}

/*
 * TODO: If a controller has multiple namespaces, they could all use the same queue.
 *  For now, give each namespace/thread combination its own queue.
 */
static int
nvme_init_ns_worker_ctx(struct ns_worker_ctx *ns_ctx)
{
        struct spdk_nvme_io_qpair_opts opts;
        struct ns_entry *entry = ns_ctx->entry;
    int i = 0;

        spdk_nvme_ctrlr_get_default_io_qpair_opts(entry->u.nvme.ctrlr, &opts, sizeof(opts));
        if (opts.io_queue_requests < entry->num_io_requests) {
                opts.io_queue_requests = entry->num_io_requests;
        }

    ns_ctx->u.nvme.qpair = calloc(g_num_pair, sizeof(struct spdk_nvme_qpair*));
    for (i = 0; i < g_num_pair; i++) {
        ns_ctx->u.nvme.qpair[i] = spdk_nvme_ctrlr_alloc_io_qpair(entry->u.nvme.ctrlr, &opts,
                       sizeof(opts));
        if (!ns_ctx->u.nvme.qpair[i]) {
            printf("ERROR: spdk_nvme_ctrlr_alloc_io_qpair failed\n");
            return -1;
        }
    }

        return 0;
}

参考代码