added directories/general structure for API.

Mapping are wrong we need physical addresses when remapping descriptors

2 files changed, 259 insertions, 0 deletions

diff --git a/drivers/rtl8169.c b/drivers/rtl8169.c
new file mode 100644
index 0000000..7218a8e
--- /dev/null
+++ b/drivers/rtl8169.c
@@ -0,0 +1,217 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <string.h>
+#include <sys/types.h>
+#include <linux/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <reg_api.h>
+#include "rtl8169.h"
+#include "vfio_api.h"
+
+typedef uint64_t dma_addr_t;
+
+const char glob_uuid[] = "d0b24768-9c51-11e7-8de5-c7b0c2769e69";
+/* iommu alignment */
+
+int print_packet(unsigned char* buffer)
+{
+	int i;
+	//unsigned int* b = (unsigned int*)buffer;
+	for (i = 0; i < 16; i++) {
+		printf("%02x", buffer[i]);
+	}
+}
+
+static inline void rtl8169_mark_to_asic(struct rxdesc *desc, uint32_t rx_buf_sz)
+{
+	uint32_t eor = le32_to_cpu(desc->opts1) & RingEnd;
+
+	/* Force memory writes to complete before releasing descriptor */
+	dma_wmb();
+
+	desc->opts1 = cpu_to_le32(DescOwn | eor | rx_buf_sz);
+}
+
+static inline void rtl8169_map_to_asic(struct rxdesc *desc, dma_addr_t mapping,
+	uint32_t rx_buf_sz)
+{
+	desc->addr = cpu_to_le64(mapping);
+	rtl8169_mark_to_asic(desc, rx_buf_sz);
+}
+
+/* drivers/net/ethernet/realtek/r8169.c */
+int rtl_8169_remap(int container, void *map, struct rxdesc *rx_ring)
+{
+
+	/* unock config registers */
+	io_write8(map + 0x50, 0xc0);
+	/* RxConfig = RXFTH: unlimited, MXDMA: unlimited, AAP: set (promisc. mode set) */
+	io_write32(map + 0x44, 0x0000e70f);
+	/* Already enabled, do we need a reset? */
+	//io_write8(map + 0x37, 0x04);
+	/* TxConfig = IFG: normal, MXDMA: unlimited */
+	io_write32(map + 0x40, 0x03000700);
+	/* Max Rx packet size */
+	io_write16(map + 0xDA, 0x1FFF);
+	/* max Tx packet size */
+	io_write8(map + 0xEC, 0x3B);
+
+	/* physical address for descriptors */
+	//io_write32(TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32);
+	//io_write32(TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32));
+
+	/* XXX need physaddress here not virtual */
+	io_write32(map + 0xE8, ((uint64_t) rx_ring) >> 32);
+	io_write32(map + 0xE4, ((uint64_t) rx_ring) & 0x00000000ffffffff);
+
+	/* Done */
+	//io_write8(map + 0x37, 0x0C); /* Enable Rx/Tx in the Command register */
+	/* Lock config registers */
+	io_write8(map + 0x50, 0x00);
+
+	return 0;
+}
+
+void rtl_8169_dump(int container, void *map)
+{
+	//printf("0x%02x\n", io_read8(map + 0x50));
+	//printf("0x%08x\n", io_read32(map + 0x44));
+	//printf("0x%02x\n", io_read8(map + 0x37));
+	//printf("0x%08x\n", io_read32(map + 0x40));
+	//printf("0x%04x\n", io_read16(map + 0xDA));
+	//printf("0x%02x\n", io_read8(map + 0xEC));
+	printf("RX LOW 0x%02x\n", io_read32(map + 0xE4));
+	printf("RX HIGH 0x%02x\n", io_read32(map + 0xE8));
+}
+
+void *rtl8169_rx_fill(int container, struct rxdesc *rx_desc_ring, uint64_t iova_start)
+{
+	int i;
+	void *vaddr = NULL;
+	int ret = 0;
+
+	ret += dma_map_type1(container, NUM_RX_DESC * 2 * 1024 * 1024, &vaddr,
+			    iova_start);
+	if (ret)
+		printf("Failed to map memory\n");
+
+	for (i = 0; i < NUM_RX_DESC; i++) {
+		rtl8169_map_to_asic(&rx_desc_ring[i], (dma_addr_t)dma_map.iova + i * 2048, 2048);
+	}
+
+	return vaddr;
+}
+
+
+void usage(char *name)
+{
+	printf("usage: %s <group id>\n", name);
+}
+
+int main(int argc, char* argv[])
+{
+	int container, group, device;
+	int ret, i;
+	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	struct vfio_iommu_type1_info iommu_info = { .argsz = sizeof(iommu_info) };
+	struct vfio_region_info region_info = { .argsz = sizeof(region_info) };
+	void *map;
+	struct rxdesc *rx_desc_ring;
+	void *rxdata;
+	int cnt = 0;
+
+	if (argc != 2) {
+		usage(argv[0]);
+		return -1;
+	}
+
+	container = get_container();
+	if (container < 0)
+		goto out;
+
+	group = get_group(atoi(argv[1]));
+	if (group < 0)
+		goto out;
+
+	/* add group to the container */
+	ret = ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);
+	if (ret) {
+		printf("Failed to set group container\n");
+		goto out;
+	}
+
+	ret = ioctl(container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
+	if (ret) {
+		printf("Failed to set IOMMU\n");
+		goto out;
+	}
+	/* Get addition IOMMU info */
+	ioctl(container, VFIO_IOMMU_GET_INFO, &iommu_info);
+
+	/* Get a file descriptor for the device */
+	device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, glob_uuid);
+	/* Test and setup the device */
+	ioctl(device, VFIO_DEVICE_GET_INFO, &device_info);
+
+	printf("Device supports %d regions, %d irqs\n",
+	       device_info.num_regions, device_info.num_irqs);
+
+	/* Alloc RX descriptors */
+	ret = dma_map_type1(container, R8169_RX_RING_BYTES, (void *)&rx_desc_ring, 0);
+	printf("Rx desc ring %p\n", rx_desc_ring);
+	/* TODO Alloc TX descriptors */
+	for (i = 0; i < device_info.num_regions; i++) {
+		region_info.index = i;
+		if (ioctl(device, VFIO_DEVICE_GET_REGION_INFO, &region_info))
+			continue;
+		if (!region_info.size) {
+			/* printf("Region:%d unimplemented PCI BAR\n", i); */
+			continue;
+		}
+
+		if (region_info.flags & VFIO_REGION_INFO_FLAG_MMAP) {
+
+			printf("## Region:%d size %lu, offset 0x%lx, flags 0x%x ##\n",
+			       i, (unsigned long)region_info.size,
+			       (unsigned long)region_info.offset, region_info.flags);
+			map = mmap(NULL, (size_t)region_info.size,
+					 PROT_READ | PROT_WRITE, MAP_SHARED,
+					 device, (off_t)region_info.offset);
+			if (map == MAP_FAILED) {
+				printf("mmap failed\n");
+				continue;
+			}
+			rtl_8169_remap(container, map, rx_desc_ring);
+			rtl_8169_dump(container, map);
+
+			munmap(map, (size_t)region_info.size);
+		}
+	}
+	rxdata = rtl8169_rx_fill(container, rx_desc_ring, R8169_RX_RING_BYTES);
+
+	while (cnt < 3) {
+		for (i = 0; i < 16; i++) {
+			int value = rx_desc_ring[i].opts1;
+			printf("packet[%03d]: size=%x  %x  %llx: ", i,
+			       (value & 0x3FFF) - 4, rx_desc_ring[i].opts2,
+			       rx_desc_ring[i].addr);
+			print_packet(rxdata + i * 2048);
+			printf("\n");
+		}
+		printf("------------------------------------------\n");
+		sleep(1);
+		cnt++;
+	}
+	dma_unmap_type1(container, R8169_RX_RING_BYTES, rx_desc_ring);
+
+	return 0;
+out:
+	return -1;
+}
diff --git a/drivers/rtl8169.h b/drivers/rtl8169.h
new file mode 100644
index 0000000..baa5f3b
--- /dev/null
+++ b/drivers/rtl8169.h
@@ -0,0 +1,42 @@
+#define TEST_SIZE	1024 * 1024
+#define NUM_TX_DESC	64	/* Number of Tx descriptor registers */
+#define NUM_RX_DESC	256U	/* Number of Rx descriptor registers */
+
+#define COMPILER_BARRIER() asm volatile("" ::: "memory")
+#define MEMORY_BARRIER() asm volatile ("mfence" ::: "memory")
+#define STORE_BARRIER() asm volatile ("sfence" ::: "memory")
+#define dma_wmb() STORE_BARRIER()
+
+#define cpu_to_le32(x) htole32(x)
+#define cpu_to_le64(x) htole64(x)
+#define le32_to_cpu(x) le32toh(x)
+
+struct Descriptor {
+	unsigned int command,  /* command/status uint32_t */
+	vlan,     /* currently unused */
+	low_buf,  /* low 32-bits of physical buffer address */
+	high_buf; /* high 32-bits of physical buffer address */
+};
+
+enum rtl_desc_bit {
+	/* First doubleword. */
+	DescOwn = (1 << 31), /* Descriptor is owned by NIC */
+	RingEnd = (1 << 30), /* End of descriptor ring */
+	FirstFrag = (1 << 29), /* First segment of a packet */
+	LastFrag = (1 << 28), /* Final segment of a packet */
+};
+
+struct txdesc {
+	__le32 opts1;
+	__le32 opts2;
+	__le64 addr;
+};
+
+struct rxdesc {
+	__le32 opts1;
+	__le32 opts2;
+	__le64 addr;
+};
+#define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct txdesc))
+#define R8169_RX_RING_BYTES	(NUM_RX_DESC * sizeof(struct rxdesc))
+