RC ---------- 808 ------------- NVIDIA
NT Up
Link Side Virtual Side
服务器通过非透明桥片与NVIDIA Xavier模组相连。
NVIDIA端申请IOVA地址。
#if 1
int ret;
int page_num=4;
struct device *cdev=&(pdev->dev.parent);
//printk("2222 cdev=0x%x\n",cdev);
struct iommu_domain *domain = iommu_get_domain_for_dev(cdev);
struct iova_domain *iovad=(struct iova_domain *)domain->iova_cookie;
struct iova *iova = NULL;
unsigned long shift, iova_len;
shift = iova_shift(iovad);
printk("shift=%d\n",shift);
iova = alloc_iova(iovad, page_num, cdev->coherent_dma_mask >> shift, 1);
（iova =iommu_dma_alloc_iova(cdev, BAR0_SIZE, cdev->coherent_dma_mask);可以用iommu_dma_alloc_iova申请iova，但是这个函数在内核中没有EXPORT——SYMBOL，所以在内核ko中使用不了，分析iommu_dma_alloc_iova函数，发现可以使用alloc_iova代替。如果不是编译内核ko，而是直接编译到内核，则可以使用iommu_dma_alloc_iova）
if (iova)
printk("GET IOVA 0x%llx\n" ,(dma_addr_t) iova->pfn_lo << shift);
else
printk("alloc iova failed!\n");
VIRTUAL_SIDE_MEM_PHYSICAL_ADDRESS_1=(dma_addr_t) iova->pfn_lo <<shift;
struct page *bar2_ram_page = alloc_pages(GFP_KERNEL, 1);
//VIRTUAL_SIDE_MEM_PHYSICAL_ADDRESS_1 = page_to_phys(bar2_ram_page);
printk("alloc RAM phys: 0x%llx\n",page_to_phys(bar2_ram_page));
ret = iommu_map(domain,(dma_addr_t) iova->pfn_lo <<shift,page_to_phys(bar2_ram_page),PAGE_SIZE,IOMMU_READ | IOMMU_WRITE);
if (ret) {
printk("iommu_map(RAM) failed: %d\n",ret);
}
// struct page *bar4_ram_page = alloc_pages(GFP_KERNEL, 1);
// VIRTUAL_SIDE_MEM_PHYSICAL_ADDRESS_2 = page_to_phys(bar4_ram_page);
// printk("alloc RAM2 phys: 0x%llx\n",VIRTUAL_SIDE_MEM_PHYSICAL_ADDRESS_2);
#else
struct device *cdev=&(pdev->dev);
dma_addr_t bar0_iova;
void *bar2_ram_map;
dma_addr_t bar2_iova;
bar2_ram_map = dma_alloc_coherent(cdev, 1024*4,&bar2_iova,GFP_KERNEL);
if(!bar2_ram_map)
printk("dma_alloc_coherent failed!\n");
else
printk("@@@bar2_ram_map=0x%llx bar2_iova=0x%llx,PHY:0x%llx\n",bar2_ram_map,bar2_iova,__pa(bar2_ram_map));
VIRTUAL_SIDE_MEM_PHYSICAL_ADDRESS_1=bar2_iova;
#endif
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
#lspci -v
0005:01:00.0 Bridge: Pericom Semiconductor Device 8608
Subsystem: Pericom Semiconductor Device 8608
Flags: bus master, fast devsel, latency 0
Memory at 1f40200000 (32-bit, non-prefetchable) [size=128K]
Memory at 1f40000000 (32-bit, non-prefetchable) [size=1M] （修改桥片bar空间大小只能烧EEPROM，X86是在BIOS中对PCI设备进行扫描）
Memory at 1f40100000 (32-bit, non-prefetchable) [size=1M]
Capabilities: [40] Power Management version 3
Capabilities: [48] MSI: Enable- Count=1/1 Maskable- 64bit+
Capabilities: [68] Express Endpoint, MSI 00
Capabilities: [a4] Subsystem: Pericom Semiconductor Device 8608
Capabilities: [c8] Vendor Specific Information: Len=38 <?>
Capabilities: [100] Device Serial Number 00-00-00-00-00-00-00-00
Capabilities: [fb4] Advanced Error Reporting
Capabilities: [138] Power Budgeting <?>
Capabilities: [148] Virtual Channel
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
验证流程（两个NVIDIA Xavier模组通过非透明桥片互联，且都设置为RC）
1.首先在link side和virtual side使用lspci命令查看是否查看到达尔的桥片。
link side如下：
root@dsp:~# lspci
0001:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad2 (rev a1)
0001:01:00.0 SATA controller: Marvell Technology Group Ltd. Device 9171 (rev 13)
0005:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad0 (rev a1)
0005:01:00.0 Bridge: Pericom Semiconductor Device 8608
virtual side如下：
root@dsp:/home/dsp/1223/driver/virtual# lspci
0005:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad0 (rev a1)
0005:01:00.0 PCI bridge: Pericom Semiconductor Device 8608
0005:02:01.0 PCI bridge: Pericom Semiconductor Device 8608
0005:03:00.0 Bridge: Pericom Semiconductor Device 8608
2.link side首先insmod pscgrande.ko
3.virtual side再insmod pscgrande.ko
4.link side 使用lspci -v查看bar2地址为1f40000000。
0005:01:00.0 Bridge: Pericom Semiconductor Device 8608
Subsystem: Pericom Semiconductor Device 8608
Flags: bus master, fast devsel, latency 0, IRQ 39
Memory at 1f40200000 (32-bit, non-prefetchable) [size=128K]
Memory at 1f40000000 (32-bit, non-prefetchable) [size=1M]
Memory at 1f40100000 (32-bit, non-prefetchable) [size=1M]
Capabilities: [40] Power Management version 3
Capabilities: [48] MSI: Enable- Count=1/1 Maskable- 64bit+
Capabilities: [68] Express Endpoint, MSI 00
Capabilities: [a4] Subsystem: Pericom Semiconductor Device 8608
Capabilities: [c8] Vendor Specific Information: Len=38 <?>
Capabilities: [100] Device Serial Number 00-00-00-00-00-00-00-00
Capabilities: [fb4] Advanced Error Reporting
Capabilities: [138] Power Budgeting <?>
Capabilities: [148] Virtual Channel
Kernel driver in use: psc8608
使用busybox devmem （0x1f40000000加上偏移0xf8000） 32 0x12345678---->0xf8000是virtual side申请的iova的低20位（iova地址为0xffff8000），高12位配置到桥片的TBR寄存器
5.virtual side
root@dsp:/home/dsp/1223/driver/virtual# busybox devmem 0x84c134000---->0x84c134000是virtual side申请的物理地址，读出来与link side写入的数据相同
0x12345678
-----------------------------------------------------------------------------------------------------桥片主要配置了TBR和BDF表-------------------------------------------------------------------------------------------------------------
link side dmesg：
[ 6097.518402] pscgrande: loading out-of-tree module taints kernel.
[ 6097.520561] pscgrande: Initialization started
[ 6097.521321] pscgrande_probe(424): pDriver 0xffffffc7b4af9000
[ 6097.521513] pscgrande_probe(436): ID = 860812D8
[ 6097.521788] pscgrande_probe(463): got pci resource: bar 0 physical address(0x1f40200000)
[ 6097.522566] pscgrande_probe(470): bar 0 physical remap to virt: 0xffffff8024940000
[ 6097.522713] pscgrande_probe(471): bar 0 length: 131072
[ 6097.522830] pscgrande_probe(479): CDEP found
[ 6097.522916] pscgrande_probe(490): got pci resource: bar 2 physical address(0x1f40000000)
[ 6097.524422] pscgrande_probe(497): bar 2 physical remap to virt: 0xffffff8024a00000
[ 6097.524921] pscgrande_probe(498): bar 2 length: 1048576
[ 6097.525214] ProbeReqID(296): ReqID(W) = 0000 [b:00 s:00 f:0]
[ 6097.525363] NtLutAdd(381): Added Req ID (0000) to LUT #0
[ 6097.525618] ProbeReqID(296): ReqID(R) = 0000 [b:00 s:00 f:0]
[ 6097.525737] NtLutAdd(381): Added Req ID (0000) to LUT #1
[ 6097.525878] pscgrande_probe(532): request_irq succeeded. irq = 39
[ 6097.526015] pscgrande_probe(541): read scatchpad 0 = 00000000
[ 6097.528817] pscgrande_probe(546): write scatchpad 0 = 860612D8
[ 6097.534019] pscgrande_probe(541): read scatchpad 1 = 00000000
[ 6097.540055] pscgrande_probe(546): write scatchpad 1 = 860612D8
[ 6097.546100] pscgrande_probe(541): read scatchpad 2 = 00000000
[ 6097.552184] pscgrande_probe(546): write scatchpad 2 = 860612D8
[ 6097.557928] pscgrande_probe(541): read scatchpad 3 = 00000000
[ 6097.564367] pscgrande_probe(546): write scatchpad 3 = 860612D8
[ 6097.570151] pscgrande_probe(541): read scatchpad 4 = 00000000
[ 6097.575982] pscgrande_probe(546): write scatchpad 4 = 860612D8
[ 6097.581674] pscgrande_probe(541): read scatchpad 5 = 00000000
[ 6097.587801] pscgrande_probe(546): write scatchpad 5 = 860612D8
[ 6097.593385] pscgrande_probe(541): read scatchpad 6 = 00000000
[ 6097.599567] pscgrande_probe(546): write scatchpad 6 = 860612D8
[ 6097.605237] pscgrande_probe(541): read scatchpad 7 = 00000000
[ 6097.611133] pscgrande_probe(546): write scatchpad 7 = 860612D8
[ 6097.617493] pscgrande_probe(579): 0xC64 = 00007FFF
[ 6097.622376] pscgrande: Class created
[ 6097.625310] pscgrande: alloc ffffffc7b4af9000
[ 6173.258553] t19x-arm-smmu 12000000.iommu: Unhandled context fault: smmu0, iova=0x00000000, fsynr=0x3c0001, cb=3, sid=91(0x5b - PCIE5), pgd=0, pud=0, pmd=0, pte=0
[ 6173.259508] mc-err: (255) csr_pcie5r: EMEM address decode error
[ 6173.259746] mc-err: status = 0x200640e2; addr = 0xffffffff00; hi_adr_reg=ff08
[ 6173.259897] mc-err: secure: yes, access-type: read
[ 6173.260018] mc-err: mcerr: unknown intr source intstatus = 0x00000000, intstatus_1 = 0x00000000
[ 6173.270876] pscgrande_cdep_interrupt(115): irq = 39 pdx 0xffffffc7b4af9000
[ 6173.271047] pscgrande_cdep_interrupt(116): isLink = 1
[ 6173.271188] pscgrande_cdep_interrupt(119): 0x11C5C = 00008000
[ 6173.271338] @@@@ TBR2 nValue=0xffff8000
[ 6173.271349] @@@@ TBR3 nValue=0x0
virtual side dmesg：
[ 6170.378112] pscgrande: loading out-of-tree module taints kernel.
[ 6170.379468] pscgrande: Initialization started
[ 6170.380008] shift=12
[ 6170.380018] GET IOVA 0xffff8000
[ 6170.380026] alloc RAM phys: 0x84c134000
[ 6170.380039] pscgrande_probe(410): pDriver 0xffffffc7d675b000
[ 6170.380179] pscgrande_probe(423): ID = 860812D8
[ 6170.380438] pscgrande_probe(450): got pci resource: bar 0 physical address(0x1f40200000)
[ 6170.380791] pscgrande_probe(457): bar 0 physical remap to virt: 0xffffff80127c0000
[ 6170.380924] pscgrande_probe(458): bar 0 length: 131072
[ 6170.381012] pscgrande_probe(466): CDEP found
[ 6170.381098] pscgrande_probe(477): got pci resource: bar 2 physical address(0x1f40000000)
[ 6170.381265] pscgrande_probe(484): bar 2 physical remap to virt: 0xffffff801c400000
[ 6170.381403] pscgrande_probe(485): bar 2 length: 1048576
[ 6170.381687] ProbeReqID(239): ReqID(W) = 0000 [b:00 s:00 f:0]
[ 6170.381801] NtLutAdd(324): Added Req ID (0000) to LUT #0
[ 6170.382029] ProbeReqID(239): ReqID(R) = 0000 [b:00 s:00 f:0]
[ 6170.382139] NtLutAdd(324): Added Req ID (0000) to LUT #1
[ 6170.382254] pscgrande_probe(521): request_irq succeeded. irq = 39
[ 6170.382392] pscgrande_probe(530): read scatchpad 0 = 860612D8
[ 6170.387682] pscgrande_probe(543): index = 0, peer was loaded already
[ 6170.395025] pscgrande: Class created
[ 6170.397814] pscgrande: alloc ffffffc7d675b000
[ 6170.409558] tegra-asoc: sound: ASoC: CODEC DAI rt5659-aif1 not registered
[ 6170.412653] tegra-asoc: sound: snd_soc_register_card failed (-517)
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------
driver.tar (20201223)是达尔提供的桥片驱动代码。
readme.txt是达尔提供的对驱动代码的说明。
driver_final文件夹是我在达尔驱动的基础上做的修改，加载driver_final驱动后，可以实现link端对virtal端的读写（devmem）。
验证流程（两个NVIDIA Xavier模组通过非透明桥片互联，且都设置为RC）
1.首先在link side和virtual side使用lspci命令查看是否查看到达尔的桥片。
link side如下：
root@dsp:~# lspci
0001:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad2 (rev a1)
0001:01:00.0 SATA controller: Marvell Technology Group Ltd. Device 9171 (rev 13)
0005:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad0 (rev a1)
0005:01:00.0 Bridge: Pericom Semiconductor Device 8608
virtual side如下：
root@dsp:/home/dsp/1223/driver/virtual# lspci
0005:00:00.0 PCI bridge: NVIDIA Corporation Device 1ad0 (rev a1)
0005:01:00.0 PCI bridge: Pericom Semiconductor Device 8608
0005:02:01.0 PCI bridge: Pericom Semiconductor Device 8608
0005:03:00.0 Bridge: Pericom Semiconductor Device 8608
2.link side首先insmod pscgrande.ko,在insmod时对端会出现如下报错信息，这是正常的，是为了获取BDF信息主动触发的。
加载ko后Link side dmesg信息如下，
3.virtual side再insmod pscgrande.ko
加载ko时会触发Link side中断，从而从scratchpad中读取virtual side申请的iova地址。Link side打印如下，
Virtual side打印如下，
4.link side 使用lspci -v查看bar2地址为1f40000000。
使用busybox devmem （0x1f40000000加上偏移0xf8000） 32 0x12345678---->0xf8000是virtual side申请的iova的低20位（iova地址为0xffff8000），高12位配置到桥片的TBR寄存器
5.virtual side
root@dsp:/home/dsp/1223/driver/virtual# busybox devmem 0x84c134000---->0x84c134000是virtual side申请的物理地址，读出来与link side写入的数据相同
0x12345678