diff --git a/platform/pddf/i2c/debian/rules b/platform/pddf/i2c/debian/rules index e1f08e0cf35..f62a480aa2f 100755 --- a/platform/pddf/i2c/debian/rules +++ b/platform/pddf/i2c/debian/rules @@ -19,7 +19,7 @@ PACKAGE_PRE_NAME := sonic-platform-pddf KVERSION ?= $(shell uname -r) KERNEL_SRC := /lib/modules/$(KVERSION) MOD_SRC_DIR:= $(shell pwd) -MODULE_DIRS:= client cpld cpld/driver cpldmux cpldmux/driver fpgai2c fpgai2c/driver fpgapci fpgapci/driver fpgapci/algos fan fan/driver mux gpio led psu psu/driver sysstatus xcvr xcvr/driver +MODULE_DIRS:= client cpld cpld/driver cpldmux cpldmux/driver fpgai2c fpgai2c/driver fpgapci fpgapci/driver fpgapci/algos multifpgapci multifpgapci/driver multifpgapci/i2c fan fan/driver mux gpio led psu psu/driver sysstatus xcvr xcvr/driver MODULE_DIR:= modules UTILS_DIR := utils SERVICE_DIR := service diff --git a/platform/pddf/i2c/modules/Makefile b/platform/pddf/i2c/modules/Makefile index a936ff9756c..a6a76821406 100644 --- a/platform/pddf/i2c/modules/Makefile +++ b/platform/pddf/i2c/modules/Makefile @@ -1 +1 @@ -obj-m := client/ cpld/ cpldmux/ fpgai2c/ fpgapci/ xcvr/ mux/ gpio/ psu/ fan/ led/ sysstatus/ +obj-m := client/ cpld/ cpldmux/ fpgai2c/ fpgapci/ multifpgapci/ xcvr/ mux/ gpio/ psu/ fan/ led/ sysstatus/ diff --git a/platform/pddf/i2c/modules/include/pddf_client_defs.h b/platform/pddf/i2c/modules/include/pddf_client_defs.h index cb26dd1ab4f..2620986dbcb 100644 --- a/platform/pddf/i2c/modules/include/pddf_client_defs.h +++ b/platform/pddf/i2c/modules/include/pddf_client_defs.h @@ -34,6 +34,7 @@ #define SYSSTATUS "PDDF_SYSSTATUS" #define XCVR "PDDF_XCVR" #define FPGA "PDDF_FPGAPCI" +#define MULTIFPGA "PDDF_MULTIFPGAPCI" #define PDDF_DEBUG diff --git a/platform/pddf/i2c/modules/include/pddf_i2c_algo.h b/platform/pddf/i2c/modules/include/pddf_i2c_algo.h index 3526b456516..01e69847655 100644 --- a/platform/pddf/i2c/modules/include/pddf_i2c_algo.h +++ b/platform/pddf/i2c/modules/include/pddf_i2c_algo.h @@ -9,7 +9,7 @@ #include "pddf_client_defs.h" /* max number of adapters */ -#define I2C_PCI_MAX_BUS 16 +#define I2C_PCI_MAX_BUS 512 /** * struct fpgapci_devdata - PCI device data structure diff --git a/platform/pddf/i2c/modules/include/pddf_led_defs.h b/platform/pddf/i2c/modules/include/pddf_led_defs.h index 15ef5ce1f91..ac15f4ce568 100644 --- a/platform/pddf/i2c/modules/include/pddf_led_defs.h +++ b/platform/pddf/i2c/modules/include/pddf_led_defs.h @@ -123,9 +123,11 @@ typedef enum{ LED_FANTRAY, LED_DIAG, LED_LOC, - LED_BMC, + LED_BMC, + LED_PORT, LED_TYPE_MAX } LED_TYPE; + char* LED_TYPE_STR[LED_TYPE_MAX] = { "LED_SYS", @@ -134,7 +136,8 @@ char* LED_TYPE_STR[LED_TYPE_MAX] = "LED_FANTRAY", "LED_DIAG", "LED_LOC", - "LED_BMC" + "LED_BMC", + "LED_PORT", }; /***************************************** diff --git a/platform/pddf/i2c/modules/include/pddf_multifpgapci_defs.h b/platform/pddf/i2c/modules/include/pddf_multifpgapci_defs.h new file mode 100644 index 00000000000..c724ae0cccc --- /dev/null +++ b/platform/pddf/i2c/modules/include/pddf_multifpgapci_defs.h @@ -0,0 +1,57 @@ +/* + * Copyright 2025 Nexthop Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Description: + * Platform MULTIFPGAPCI defines/structures header file + */ + +#ifndef __PDDF_MULTIFPGAPCI_DEFS_H__ +#define __PDDF_MULTIFPGAPCI_DEFS_H__ + +#include "linux/types.h" +#include + +#include "pddf_multifpgapci_i2c_defs.h" + +#define NAME_SIZE 32 +#define KOBJ_FREE(obj) \ + if (obj) \ + kobject_put(obj); + +struct pddf_multifpgapci_drvdata { + struct pci_dev *pci_dev; + resource_size_t bar_start; + void *__iomem fpga_data_base_addr; + // i2c + size_t bar_length; + struct kobject *i2c_kobj; + struct i2c_adapter_drvdata i2c_adapter_drvdata; + bool i2c_adapter_drvdata_initialized; +}; + +// FPGA +typedef struct { + uint32_t data_base_offset; + uint32_t data_size; +} FPGA_OPS_DATA; + +struct pddf_multi_fpgapci_ops_t { + int (*post_device_operation)(struct pci_dev *); +}; + +extern struct pddf_multi_fpgapci_ops_t pddf_multi_fpgapci_ops; + +extern int (*ptr_multifpgapci_readpci)(struct pci_dev *, uint32_t, uint32_t *); +extern int (*ptr_multifpgapci_writepci)(struct pci_dev *, uint32_t, uint32_t); + +#endif diff --git a/platform/pddf/i2c/modules/include/pddf_multifpgapci_i2c_defs.h b/platform/pddf/i2c/modules/include/pddf_multifpgapci_i2c_defs.h new file mode 100644 index 00000000000..a2618f38338 --- /dev/null +++ b/platform/pddf/i2c/modules/include/pddf_multifpgapci_i2c_defs.h @@ -0,0 +1,73 @@ +/* + * Copyright 2025 Nexthop Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __PDDF_MULTIFPGAPCI_I2C_DEFS_H__ +#define __PDDF_MULTIFPGAPCI_I2C_DEFS_H__ + +#include +#include "linux/types.h" +#include +#include +#include + +#include "pddf_client_defs.h" + +#define I2C_PCI_MAX_BUS 512 + +struct i2c_adapter_attrs { + PDDF_ATTR attr_virt_bus; + PDDF_ATTR attr_ch_base_offset; + PDDF_ATTR attr_ch_size; + PDDF_ATTR attr_num_virt_ch; + PDDF_ATTR attr_new_i2c_adapter; + PDDF_ATTR attr_del_i2c_adapter; +}; + +#define NUM_I2C_ADAPTER_ATTRS \ + (sizeof(struct i2c_adapter_attrs) / sizeof(PDDF_ATTR)) + +struct i2c_adapter_sysfs_vals { + uint32_t virt_bus; + uint32_t ch_base_offset; + uint32_t ch_size; + uint32_t num_virt_ch; +}; + +struct i2c_adapter_drvdata { + // temp_sysfs_vals store temporary values provided by sysfs, + // which are eventually copied/saved to I2C adapter platform data. + struct i2c_adapter_sysfs_vals temp_sysfs_vals; + + // platform data + struct i2c_adapter i2c_adapters[I2C_PCI_MAX_BUS]; + bool i2c_adapter_registered[I2C_PCI_MAX_BUS]; + int virt_bus; + void *__iomem ch_base_addr; + int ch_size; + int num_virt_ch; + + // sysfs attrs + struct i2c_adapter_attrs attrs; + struct attribute *i2c_adapter_attrs[NUM_I2C_ADAPTER_ATTRS + 1]; + struct attribute_group i2c_adapter_attr_group; +}; + +extern int pddf_multifpgapci_i2c_module_init(struct pci_dev *pci_dev, + struct kobject *kobj); + +// Only called if multifpgapci_i2c_module_init succeeded +extern void pddf_multifpgapci_i2c_module_exit(struct pci_dev *pci_dev, + struct kobject *kobj); + +#endif \ No newline at end of file diff --git a/platform/pddf/i2c/modules/led/pddf_led_module.c b/platform/pddf/i2c/modules/led/pddf_led_module.c index aa55bb49d97..b3d9433b19c 100644 --- a/platform/pddf/i2c/modules/led/pddf_led_module.c +++ b/platform/pddf/i2c/modules/led/pddf_led_module.c @@ -23,15 +23,17 @@ #include #include #include -#include "pddf_led_defs.h" -#include "pddf_client_defs.h" #include #include #include +#include "pddf_client_defs.h" +#include "pddf_led_defs.h" +#include "pddf_multifpgapci_defs.h" #define DEBUG 0 #define MAX_PSU_NUM 2 #define MAX_FANTRAY_NUM 6 +#define MAX_PORT_NUM (48 * 8) LED_OPS_DATA sys_led_ops_data[1]={0}; LED_OPS_DATA psu_led_ops_data[MAX_PSU_NUM]={0}; LED_OPS_DATA diag_led_ops_data[1]= {0}; @@ -39,6 +41,7 @@ LED_OPS_DATA fan_led_ops_data[1]= {0}; LED_OPS_DATA loc_led_ops_data[1]= {0}; LED_OPS_DATA bmc_led_ops_data[1]= {0}; LED_OPS_DATA fantray_led_ops_data[MAX_FANTRAY_NUM]={0}; +LED_OPS_DATA port_led_ops_data[MAX_PORT_NUM] = {0}; LED_OPS_DATA temp_data={0}; LED_OPS_DATA* dev_list[LED_TYPE_MAX] = { sys_led_ops_data, @@ -48,6 +51,7 @@ LED_OPS_DATA* dev_list[LED_TYPE_MAX] = { diag_led_ops_data, loc_led_ops_data, bmc_led_ops_data, + port_led_ops_data, NULL }; int num_psus = 0; @@ -63,6 +67,8 @@ extern ssize_t store_pddf_data(struct device *dev, struct device_attribute *da, extern ssize_t show_pddf_s3ip_data(struct device *dev, struct device_attribute *da, char *buf); extern ssize_t store_pddf_s3ip_data(struct device *dev, struct device_attribute *da, const char *buf, size_t count); +extern void* get_device_table(char *name); + static LED_STATUS find_state_index(const char* state_str) { int index; char *ptr = (char *)state_str; @@ -91,6 +97,8 @@ static LED_TYPE get_dev_type(char* name) ret = LED_LOC; } else if(strstr(name, "FANTRAY_LED")) { ret = LED_FANTRAY; + } else if(strstr(name, "PORT_LED")) { + ret = LED_PORT; } #if DEBUG > 1 pddf_dbg(LED, KERN_INFO "LED get_dev_type: %s; %d\n", name, ret); @@ -111,6 +119,9 @@ static int dev_index_check(LED_TYPE type, int index) case LED_FANTRAY: if(index >= MAX_FANTRAY_NUM) return (-1); break; + case LED_PORT: + if(index >= MAX_PORT_NUM) return (-1); + break; default: if(index >= 1) return (-1); break; @@ -160,6 +171,42 @@ static void print_led_data(LED_OPS_DATA *ptr, LED_STATUS state) } } +int led_multifpgapci_read(LED_OPS_DATA *ops_ptr, uint32_t *output) { + struct pci_dev *pci_dev = NULL; + + if (ptr_multifpgapci_readpci == NULL) { + printk(KERN_ERR "PDDF_LED: pddf_multifpgapci_module is not loaded"); + return -1; + } + + pci_dev = (struct pci_dev *)get_device_table(ops_ptr->attr_devname); + if (pci_dev == NULL) { + printk(KERN_ERR "PDDF_LED: Unable to get pci_dev of %s for %s\n", + ops_ptr->attr_devname, ops_ptr->device_name); + return -1; + } + return ptr_multifpgapci_readpci( + pci_dev, ops_ptr->swpld_addr + ops_ptr->swpld_addr_offset, output); +} + +int led_multifpgapci_write(LED_OPS_DATA *ops_ptr, uint32_t val) { + struct pci_dev *pci_dev = NULL; + + if (ptr_multifpgapci_readpci == NULL || ptr_multifpgapci_writepci == NULL) { + printk(KERN_ERR "PDDF_LED: pddf_multifpgapci_module is not loaded"); + return -1; + } + + pci_dev = (struct pci_dev *)get_device_table(ops_ptr->attr_devname); + if (pci_dev == NULL) { + printk(KERN_ERR "PDDF_LED: Unable to get pci_dev of %s for %s\n", + ops_ptr->attr_devname, ops_ptr->device_name); + return -1; + } + return ptr_multifpgapci_writepci( + pci_dev, val, ops_ptr->swpld_addr + ops_ptr->swpld_addr_offset); +} + ssize_t get_status_led(struct device_attribute *da) { int ret=0; @@ -185,35 +232,51 @@ ssize_t get_status_led(struct device_attribute *da) if (strcmp(ops_ptr->attr_devtype, "cpld") == 0) { cpld_type = 1; sys_val = board_i2c_cpld_read(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + if (sys_val < 0) { + pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s 0x%x:0x%x read failed\n",__func__, + ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype, ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + return sys_val; + } } else if (strcmp(ops_ptr->attr_devtype, "fpgai2c") == 0) { sys_val = board_i2c_fpga_read(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + if (sys_val < 0) { + pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s 0x%x:0x%x read failed\n",__func__, + ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype, ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + return sys_val; + } + } else if (strcmp(ops_ptr->attr_devtype, "multifpgapci") == 0) { + ret = led_multifpgapci_read(ops_ptr, &sys_val); + if (ret) + goto ret; } else { pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s 0x%x:0x%x not configured\n",__func__, ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype, ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); return (-1); } - if (sys_val < 0) { - pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s 0x%x:0x%x read failed\n",__func__, - ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype, ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); - return sys_val; - } - strcpy(temp_data.cur_state.color, "None"); for (state=0; statedata[state].bits.mask_bits); for (j = 0; j < VALUE_SIZE && ops_ptr->data[state].reg_values[j] != 0xff; j++) { if ((color_val ^ (ops_ptr->data[state].reg_values[j] << ops_ptr->data[state].bits.pos)) == 0) { strcpy(temp_data.cur_state.color, LED_STATUS_STR[state]); - break; + goto found_match; } } } + +found_match: #if DEBUG pddf_dbg(LED, KERN_ERR "Get : %s:%d addr/offset:0x%x; 0x%x devtype:%s;%s value=0x%x [%s]\n", ops_ptr->device_name, ops_ptr->index, ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset, ops_ptr->attr_devtype, cpld_type? "cpld": "fpgai2c", sys_val, temp_data.cur_state.color); #endif + +ret: + if (ret) { + printk(KERN_ERR "%s: Error status = %d", __FUNCTION__, ret); + } + return(ret); } @@ -253,17 +316,22 @@ ssize_t set_status_led(struct device_attribute *da) if (strcmp(ops_ptr->data[cur_state].attr_devtype, "cpld") == 0) { cpld_type = 1; sys_val = board_i2c_cpld_read(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + if (sys_val < 0) + return sys_val; } else if (strcmp(ops_ptr->data[cur_state].attr_devtype, "fpgai2c") == 0) { sys_val = board_i2c_fpga_read(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + if (sys_val < 0) + return sys_val; + } else if (strcmp(ops_ptr->attr_devtype, "multifpgapci") == 0) { + ret = led_multifpgapci_read(ops_ptr, &sys_val); + if (ret) + goto ret; } else { pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s not configured\n",__func__, ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype); return (-1); } - if (sys_val < 0) - return sys_val; - new_val = (sys_val & ops_ptr->data[cur_state].bits.mask_bits) | (ops_ptr->data[cur_state].reg_values[0] << ops_ptr->data[cur_state].bits.pos); @@ -279,6 +347,13 @@ ssize_t set_status_led(struct device_attribute *da) } else if (strcmp(ops_ptr->data[cur_state].attr_devtype, "fpgai2c") == 0) { ret = board_i2c_fpga_write(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset, (uint8_t)new_val); read_val = board_i2c_fpga_read(ops_ptr->swpld_addr, ops_ptr->swpld_addr_offset); + } else if (strcmp(ops_ptr->attr_devtype, "multifpgapci") == 0) { + ret = led_multifpgapci_write(ops_ptr, new_val); + if (ret) + goto ret; + ret = led_multifpgapci_read(ops_ptr, &read_val); + if (ret) + goto ret; } else { pddf_dbg(LED, KERN_ERR "ERROR %s: %s %d devtype:%s not configured\n",__func__, ops_ptr->device_name, ops_ptr->index, ops_ptr->attr_devtype); @@ -291,6 +366,11 @@ ssize_t set_status_led(struct device_attribute *da) cpld_type? "cpld":"fpgai2c", ret, read_val, ops_ptr->data[cur_state].attr_devtype); #endif +ret: + if (ret) { + printk(KERN_ERR "%s: Error status = %d", __FUNCTION__, ret); + } + return(ret); } @@ -634,7 +714,7 @@ ssize_t store_config_data(struct device *dev, struct device_attribute *da, const ssize_t store_bits_data(struct device *dev, struct device_attribute *da, const char *buf, size_t count) { int len = 0, num1 = 0, num2 = 0, i=0, rc1=0, rc2=0; - char mask=0xFF; + unsigned int mask = 0xFFFFFFFF; char *pptr=NULL; char bits[NAME_SIZE]; struct pddf_data_attribute *ptr = (struct pddf_data_attribute *)da; @@ -853,7 +933,6 @@ static int __init led_init(void) { return (0); } - static void __exit led_exit(void) { pddf_dbg(LED, "PDDF GENERIC LED MODULE exit..\n"); free_kobjs(); diff --git a/platform/pddf/i2c/modules/multifpgapci/Makefile b/platform/pddf/i2c/modules/multifpgapci/Makefile new file mode 100644 index 00000000000..697b7878cb9 --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/Makefile @@ -0,0 +1,4 @@ +obj-m := driver/ i2c/ +obj-m += pddf_multifpgapci_module.o + +ccflags-y:= -I$(M)/modules/include diff --git a/platform/pddf/i2c/modules/multifpgapci/driver/Makefile b/platform/pddf/i2c/modules/multifpgapci/driver/Makefile new file mode 100644 index 00000000000..4a9275288f1 --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/driver/Makefile @@ -0,0 +1,4 @@ +TARGET = pddf_multifpgapci_driver +obj-m := $(TARGET).o + +ccflags-y := -I$(M)/modules/include diff --git a/platform/pddf/i2c/modules/multifpgapci/driver/pddf_multifpgapci_driver.c b/platform/pddf/i2c/modules/multifpgapci/driver/pddf_multifpgapci_driver.c new file mode 100644 index 00000000000..4cdd51aa11f --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/driver/pddf_multifpgapci_driver.c @@ -0,0 +1,696 @@ +/* + * Copyright 2025 Nexthop Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * A PDDF kernel driver for managing the creation of i2c adapters and + * various IP blocks in a system with multiple PCI FPGAs. + * Borrows from the PDDF fpgapci module by Broadcom. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pddf_client_defs.h" +#include "pddf_multifpgapci_defs.h" +#include "pddf_multifpgapci_i2c_defs.h" + +#define BDF_NAME_SIZE 32 +#define DEVICE_NAME_SIZE 32 +#define DEBUG 0 +#define DRIVER_NAME "pddf_multifpgapci" +#define MAX_PCI_NUM_BARS 6 +#define KOBJ_FREE(obj) \ + if (obj) \ + kobject_put(obj); + +extern void add_device_table(char *name, void *ptr); +extern void* get_device_table(char *name); +extern void delete_device_table(char *name); + +static bool driver_registered = false; +struct pci_driver pddf_multifpgapci_driver; +static struct kobject *multifpgapci_kobj = NULL; + +int default_multifpgapci_readpci(struct pci_dev *pci_dev, uint32_t offset, + uint32_t *output); +int default_multifpgapci_writepci(struct pci_dev *pci_dev, uint32_t val, + uint32_t offset); + +int (*ptr_multifpgapci_readpci)(struct pci_dev *, + uint32_t, uint32_t *) = default_multifpgapci_readpci; +int (*ptr_multifpgapci_writepci)(struct pci_dev *, uint32_t, + uint32_t) = default_multifpgapci_writepci; +EXPORT_SYMBOL(ptr_multifpgapci_readpci); +EXPORT_SYMBOL(ptr_multifpgapci_writepci); + +struct pddf_attrs { + PDDF_ATTR attr_dev_ops; +}; + +struct pddf_multi_fpgapci_ops_t pddf_multi_fpgapci_ops; + +EXPORT_SYMBOL(pddf_multi_fpgapci_ops); + +struct fpga_data_node { + char bdf[BDF_NAME_SIZE]; + char dev_name[DEVICE_NAME_SIZE]; // device_name as defined in pddf-device.json. + struct kobject *kobj; + struct pci_dev *dev; + struct pddf_attrs attrs; + void __iomem *fpga_ctl_addr; + unsigned long bar_start; + struct list_head list; +}; + +LIST_HEAD(fpga_list); + +struct mutex fpga_list_lock; + +static ssize_t dev_operation(struct device *dev, struct device_attribute *da, + const char *buf, size_t count); +static int map_bars(const char *bdf, + struct pddf_multifpgapci_drvdata *pci_privdata, + struct pci_dev *dev); +static void map_entire_bar(unsigned long barStart, unsigned long barLen, + struct pddf_multifpgapci_drvdata *pci_privdata, + struct fpga_data_node *fpga_data, + struct i2c_adapter_drvdata *i2c_privdata); +static int pddf_multifpgapci_probe(struct pci_dev *dev, + const struct pci_device_id *id); +static void pddf_multifpgapci_remove(struct pci_dev *dev); +static void free_bars(struct pddf_multifpgapci_drvdata *pci_privdata, + struct pci_dev *dev); + +int default_multifpgapci_readpci(struct pci_dev *pci_dev, uint32_t offset, + uint32_t *output) +{ + if (!pci_dev) { + pddf_dbg(MULTIFPGA, KERN_ERR "%s pci_dev is NULL\n", + __FUNCTION__); + return -ENODEV; + } + + struct pddf_multifpgapci_drvdata *pci_drvdata = + dev_get_drvdata(&pci_dev->dev); + *output = ioread32(pci_drvdata->fpga_data_base_addr + offset); + + return 0; +} + +int default_multifpgapci_writepci(struct pci_dev *pci_dev, uint32_t val, + uint32_t offset) +{ + if (!pci_dev) { + pddf_dbg(MULTIFPGA, KERN_ERR "%s pci_dev is NULL\n", + __FUNCTION__); + return -ENODEV; + } + + struct pddf_multifpgapci_drvdata *pci_drvdata = + dev_get_drvdata(&pci_dev->dev); + iowrite32(val, pci_drvdata->fpga_data_base_addr + offset); + + return 0; +} + +void free_pci_drvdata(struct pci_dev *pci_dev) +{ + struct pddf_multifpgapci_drvdata *pci_drvdata = + pci_get_drvdata(pci_dev); + if (!pci_drvdata) + return; + + if (pci_drvdata->i2c_adapter_drvdata_initialized) { + pddf_multifpgapci_i2c_module_exit(pci_dev, + pci_drvdata->i2c_kobj); + } + KOBJ_FREE(pci_drvdata->i2c_kobj); + + pci_set_drvdata(pci_dev, NULL); +} + +void free_sysfs_attr_groups(struct fpga_data_node *node) +{ + sysfs_remove_group(node->kobj, &pddf_clients_data_group); +} + +void delete_fpga_data_node(const char *bdf) +{ + struct fpga_data_node *node, *tmp; + + mutex_lock(&fpga_list_lock); + + list_for_each_entry_safe(node, tmp, &fpga_list, list) { + if (strcmp(node->bdf, bdf) == 0) { + KOBJ_FREE(node->kobj); + + free_pci_drvdata(node->dev); + free_sysfs_attr_groups(node); + + list_del(&node->list); + kfree(node); + break; + } + } + + mutex_unlock(&fpga_list_lock); +} + +void delete_all_fpga_data_nodes(void) +{ + struct fpga_data_node *node, *tmp; + + mutex_lock(&fpga_list_lock); + + list_for_each_entry_safe(node, tmp, &fpga_list, list) { + KOBJ_FREE(node->kobj); + + free_pci_drvdata(node->dev); + free_sysfs_attr_groups(node); + + list_del(&node->list); + kfree(node); + } + + mutex_unlock(&fpga_list_lock); +} + +struct fpga_data_node *get_fpga_data_node(const char *bdf) +{ + struct fpga_data_node *node; + struct fpga_data_node *found_node = NULL; + + mutex_lock(&fpga_list_lock); + + list_for_each_entry(node, &fpga_list, list) { + if (strcmp(node->bdf, bdf) == 0) { + found_node = node; + break; + } + } + + mutex_unlock(&fpga_list_lock); + + return found_node; +} + +void __iomem *get_fpga_ctl_addr_impl(const char *bdf) +{ + struct fpga_data_node *node = get_fpga_data_node(bdf); + + if (!node) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] No matching fpga data node\n", + __FUNCTION__); + return NULL; + } + + return node->fpga_ctl_addr; +} + +void __iomem *(*get_fpga_ctl_addr)(const char *bdf) = get_fpga_ctl_addr_impl; + +EXPORT_SYMBOL(get_fpga_ctl_addr); + +static int pddf_pci_add_fpga(char *bdf, struct pci_dev *dev) +{ + int ret = 0; + struct pddf_multifpgapci_drvdata *pci_drvdata = + dev_get_drvdata(&dev->dev); + struct fpga_data_node *fpga_data = + kzalloc(sizeof(*fpga_data), GFP_KERNEL); + if (!fpga_data) { + return -ENOMEM; + } + + strscpy(fpga_data->bdf, bdf, NAME_SIZE); + + fpga_data->kobj = kobject_create_and_add(bdf, multifpgapci_kobj); + + if (!fpga_data->kobj) { + pddf_dbg(MULTIFPGA, KERN_ERR "%s create fpga kobj failed\n", + __FUNCTION__); + ret = -ENOMEM; + goto free_fpga_data; + } + + pci_drvdata->i2c_kobj = kobject_create_and_add("i2c", fpga_data->kobj); + if (!pci_drvdata->i2c_kobj) { + pddf_dbg(MULTIFPGA, KERN_ERR "[%s] create i2c kobj failed\n", + __FUNCTION__); + ret = -ENOMEM; + goto free_fpga_kobj; + } + + ret = pddf_multifpgapci_i2c_module_init(dev, pci_drvdata->i2c_kobj); + if (ret) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] create i2c module failed %d\n", + __FUNCTION__, ret); + goto free_i2c_kobj; + } + pci_drvdata->i2c_adapter_drvdata_initialized = true; + + fpga_data->dev = dev; + + PDDF_DATA_ATTR( + dev_ops, S_IWUSR | S_IRUGO, NULL, dev_operation, + PDDF_CHAR, NAME_SIZE, NULL, (void *)&pddf_data); + + fpga_data->attrs.attr_dev_ops = attr_dev_ops; + + struct attribute *attrs_fpgapci[] = { + &fpga_data->attrs.attr_dev_ops.dev_attr.attr, + NULL, + }; + + struct attribute_group attr_group_fpgapci = { + .attrs = attrs_fpgapci, + }; + + mutex_lock(&fpga_list_lock); + list_add(&fpga_data->list, &fpga_list); + mutex_unlock(&fpga_list_lock); + + ret = sysfs_create_group(fpga_data->kobj, &attr_group_fpgapci); + + if (ret) { + pddf_dbg(MULTIFPGA, + KERN_ERR "%s create fpga sysfs attributes failed\n", + __FUNCTION__); + return ret; + } + + ret = sysfs_create_group(fpga_data->kobj, &pddf_clients_data_group); + if (ret) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] sysfs_create_group failed: %d\n", + __FUNCTION__, ret); + goto free_fpga_attr_group; + } + + return 0; + +free_fpga_attr_group: + sysfs_remove_group(fpga_data->kobj, &attr_group_fpgapci); + +free_i2c_kobj: + kobject_put(pci_drvdata->i2c_kobj); + +free_fpga_kobj: + kobject_put(fpga_data->kobj); + +free_fpga_data: + kfree(fpga_data); + + return ret; +} + +ssize_t dev_operation(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + PDDF_ATTR *ptr = (PDDF_ATTR *)da; + NEW_DEV_ATTR *cdata = (NEW_DEV_ATTR *)(ptr->data); + struct pci_dev *pci_dev = NULL; + + if (strncmp(buf, "fpgapci_init", strlen("fpgapci_init")) == 0) { + int err = 0; + struct pddf_multifpgapci_drvdata *pci_privdata = 0; + const char *bdf = dev->kobj.name; + + struct fpga_data_node *fpga_node = get_fpga_data_node(bdf); + if (!fpga_node) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] no matching fpga data node\n", + __FUNCTION__); + return -ENODEV; + } + if (cdata->i2c_name[0] == 0) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] no i2c_name specified\n", + __FUNCTION__); + return -EINVAL; + } + + pddf_dbg(MULTIFPGA, KERN_INFO "Initializing %s as %s\n", cdata->i2c_name, bdf); + strscpy(fpga_node->dev_name, cdata->i2c_name, sizeof(fpga_node->dev_name)); + + // Save pci_dev to hash table for clients to use. + pci_dev = fpga_node->dev; + add_device_table(fpga_node->dev_name, (void *)pci_dev_get(pci_dev)); + + pci_privdata = + (struct pddf_multifpgapci_drvdata *)dev_get_drvdata( + &pci_dev->dev); + + if (map_bars(bdf, pci_privdata, pci_dev)) { + pddf_dbg(MULTIFPGA, KERN_ERR "error_map_bars\n"); + pci_release_regions(pci_dev); + } + + if (pddf_multi_fpgapci_ops.post_device_operation) { + pddf_dbg(MULTIFPGA, + KERN_INFO + "[%s] Invoking post_device_operation\n", + __FUNCTION__); + err = pddf_multi_fpgapci_ops.post_device_operation( + pci_dev); + if (err) { + pddf_dbg( + MULTIFPGA, + KERN_ERR + "[%s] post_device_operation failed with error %d\n", + __FUNCTION__, err); + } + } + } else if (strncmp(buf, "fpgapci_deinit", strlen("fpgapci_deinit")) == 0) { + if (cdata->i2c_name[0] == 0) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] no i2c_name specified\n", + __FUNCTION__); + return -EINVAL; + } + + pci_dev = (struct pci_dev *)get_device_table(cdata->i2c_name); + if (pci_dev) { + delete_device_table(cdata->i2c_name); + pci_dev_put(pci_dev); + } + } + + return count; +} + +static int pddf_pci_config_data(struct pci_dev *dev) +{ + unsigned short vendorId = 0xFFFF, deviceId = 0xFFFF; + char revisionId = 0xFF, classDev = 0xFF, classProg = 0xFF; + char irqLine = 0xFF, irqPin = 0xFF; + + pddf_dbg(MULTIFPGA, KERN_INFO "[%s] PCI Config Data\n", __FUNCTION__); + + // Accessing the configuration region of the PCI device + pci_read_config_word(dev, PCI_VENDOR_ID, &vendorId); + pci_read_config_word(dev, PCI_DEVICE_ID, &deviceId); + pci_read_config_byte(dev, PCI_REVISION_ID, &revisionId); + pci_read_config_byte(dev, PCI_CLASS_PROG, &classProg); + pci_read_config_byte(dev, PCI_CLASS_DEVICE, &classDev); + + pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irqPin); + if (pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irqLine)) { + pddf_dbg(MULTIFPGA, KERN_ERR "\tPCI_INTERRUPT_LINE Error\n"); + } + + pddf_dbg(MULTIFPGA, + KERN_INFO + "\t[venId, devId]=[0x%x;0x%x] [group, class]=[%x;%x]\n", + vendorId, deviceId, classProg, classDev); + pddf_dbg(MULTIFPGA, + KERN_INFO "\trevisionId=0x%x, irq_line=0x%x, irq_support=%s\n", + revisionId, irqLine, (irqPin == 0) ? "No" : "Yes"); + + return 0; +} + +static int map_bars(const char *bdf, + struct pddf_multifpgapci_drvdata *pci_privdata, + struct pci_dev *dev) +{ + unsigned long barFlags, barStart, barEnd, barLen; + int i; + + struct fpga_data_node *fpga_node = get_fpga_data_node(bdf); + if (!fpga_node) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] No matching fpga data node\n", + __FUNCTION__); + return -ENODEV; + } + + int FPGAPCI_BAR_INDEX = -1; + + for (i = 0; i < MAX_PCI_NUM_BARS; i++) { + if ((barLen = pci_resource_len(dev, i)) != 0 && + (barStart = pci_resource_start(dev, i)) != 0) { + barFlags = pci_resource_flags(dev, i); + barEnd = pci_resource_end(dev, i); + pddf_dbg( + MULTIFPGA, + KERN_INFO + "[%s] PCI_BASE_ADDRESS_%d 0x%08lx-0x%08lx bar_len=0x%lx" + " flags 0x%08lx IO_mapped=%s Mem_mapped=%s\n", + __FUNCTION__, i, barStart, barEnd, barLen, + barFlags, + (barFlags & IORESOURCE_IO) ? "Yes" : "No", + (barFlags & IORESOURCE_MEM) ? "Yes" : "No"); + FPGAPCI_BAR_INDEX = i; + break; + } + } + + struct i2c_adapter_drvdata *i2c_privdata = + &pci_privdata->i2c_adapter_drvdata; + if (FPGAPCI_BAR_INDEX != -1) { + map_entire_bar(barStart, barLen, pci_privdata, fpga_node, + i2c_privdata); + fpga_node->bar_start = barStart; + pci_privdata->bar_start = barStart; + } else { + pddf_dbg(MULTIFPGA, KERN_INFO "[%s] Failed to find BAR\n", + __FUNCTION__); + return -1; + } + + pddf_dbg( + MULTIFPGA, + KERN_INFO + "[%s] fpga_ctl_addr:0x%p fpga_data__base_addr:0x%p" + " bar_index[%d] fpgapci_bar_len:0x%08lx fpga_i2c_ch_base_addr:0x%p supported_i2c_ch=%d" + " barStart: 0x%08lx\n", + __FUNCTION__, fpga_node->fpga_ctl_addr, + pci_privdata->fpga_data_base_addr, FPGAPCI_BAR_INDEX, + pci_privdata->bar_length, i2c_privdata->ch_base_addr, + i2c_privdata->num_virt_ch, barStart); + + return 0; +} + +static void map_entire_bar(unsigned long barStart, unsigned long barLen, + struct pddf_multifpgapci_drvdata *pci_privdata, + struct fpga_data_node *fpga_node, + struct i2c_adapter_drvdata *i2c_privdata) +{ + void __iomem *bar_base; + + bar_base = ioremap_cache(barStart, barLen); + + pci_privdata->bar_length = barLen; + pci_privdata->fpga_data_base_addr = bar_base; + fpga_node->fpga_ctl_addr = pci_privdata->fpga_data_base_addr; + + // i2c specific data store + struct i2c_adapter_sysfs_vals *i2c_pddf_data = + &i2c_privdata->temp_sysfs_vals; + + i2c_privdata->virt_bus = i2c_pddf_data->virt_bus; + i2c_privdata->ch_base_addr = bar_base + i2c_pddf_data->ch_base_offset; + i2c_privdata->num_virt_ch = i2c_pddf_data->num_virt_ch; + i2c_privdata->ch_size = i2c_pddf_data->ch_size; +} + +static void free_bars(struct pddf_multifpgapci_drvdata *pci_privdata, + struct pci_dev *dev) +{ + struct i2c_adapter_drvdata *i2c_privdata = + &pci_privdata->i2c_adapter_drvdata; + pci_iounmap(dev, (void __iomem *)pci_privdata->bar_start); + pci_privdata->fpga_data_base_addr = NULL; + i2c_privdata->ch_base_addr = NULL; +} + +static int pddf_multifpgapci_probe(struct pci_dev *dev, + const struct pci_device_id *id) +{ + struct pddf_multifpgapci_drvdata *pci_privdata = 0; + int err = 0; + pddf_dbg(MULTIFPGA, KERN_INFO "[%s]\n", __FUNCTION__); + + // TODO: Use pci_name(dev) instead? + unsigned int domain_number = pci_domain_nr(dev->bus); + unsigned int bus_number = dev->bus->number; + unsigned int device_number = PCI_SLOT(dev->devfn); + unsigned int function_number = PCI_FUNC(dev->devfn); + + char bdf[BDF_NAME_SIZE]; + snprintf(bdf, BDF_NAME_SIZE, "%04x:%02x:%02x.%x", domain_number, + bus_number, device_number, function_number); + pddf_dbg(MULTIFPGA, KERN_INFO "[%s] Probed FPGA with bdf: %s\n", + __FUNCTION__, bdf); + + if ((err = pci_enable_device(dev))) { + pddf_dbg(MULTIFPGA, + KERN_ERR + "[%s] pci_enable_device failed. dev:%s err:%#x\n", + __FUNCTION__, pci_name(dev), err); + return err; + } + + // Enable DMA + pci_set_master(dev); + + // Request MMIO/IOP resources - reserve PCI I/O and memory resources + // DRIVER_NAME shows up in /proc/iomem + if ((err = pci_request_regions(dev, DRIVER_NAME)) < 0) { + pddf_dbg(MULTIFPGA, + KERN_ERR + "[%s] pci_request_regions failed. dev:%s err:%#x\n", + __FUNCTION__, pci_name(dev), err); + goto error_pci_req; + } + + pci_privdata = + kzalloc(sizeof(struct pddf_multifpgapci_drvdata), GFP_KERNEL); + + if (!pci_privdata) { + pddf_dbg(MULTIFPGA, + KERN_ERR + "[%s] couldn't allocate pci_privdata memory\n", + __FUNCTION__); + goto error_pci_req; + } + + pci_privdata->pci_dev = dev; + pci_set_drvdata(dev, pci_privdata); + pddf_pci_config_data(dev); + pddf_pci_add_fpga(bdf, dev); + + return 0; + +// ERROR HANDLING +error_pci_req: + pci_disable_device(dev); + return -ENODEV; +} + +// Initialize the driver module (but not any device) and register +// the module with the kernel PCI subsystem. +int pddf_multifpgapci_register(const struct pci_device_id *ids, + struct kobject *kobj) +{ + int ret; + + pddf_multifpgapci_driver.name = DRIVER_NAME; + pddf_multifpgapci_driver.id_table = ids; + pddf_multifpgapci_driver.probe = pddf_multifpgapci_probe; + pddf_multifpgapci_driver.remove = pddf_multifpgapci_remove; + + if (!driver_registered) { + multifpgapci_kobj = kobject_get(kobj); + } + + ret = pci_register_driver(&pddf_multifpgapci_driver); + if (ret != 0) { + pddf_dbg(MULTIFPGA, KERN_ERR "%s: Failed to register driver\n", + __FUNCTION__); + return -EINVAL; + } + + driver_registered = true; + return ret; +} + +EXPORT_SYMBOL(pddf_multifpgapci_register); + +static void pddf_multifpgapci_remove(struct pci_dev *dev) +{ + struct pddf_multifpgapci_drvdata *pci_privdata = 0; + + if (dev == 0) { + pddf_dbg(MULTIFPGA, KERN_ERR "[%s]: dev is 0\n", __FUNCTION__); + return; + } + + pci_privdata = + (struct pddf_multifpgapci_drvdata *)dev_get_drvdata(&dev->dev); + + if (pci_privdata == 0) { + pddf_dbg(MULTIFPGA, KERN_ERR "[%s]: pci_privdata is 0\n", + __FUNCTION__); + return; + } + + delete_fpga_data_node(pci_name(dev)); + free_bars(pci_privdata, dev); + pci_disable_device(dev); + pci_release_regions(dev); + kfree(pci_privdata); +} + +int __init pddf_multifpgapci_driver_init(void) +{ + pddf_dbg(MULTIFPGA, KERN_INFO "%s ..\n", __FUNCTION__); + mutex_init(&fpga_list_lock); + return 0; +} + +void __exit pddf_multifpgapci_driver_exit(void) +{ + pddf_dbg(MULTIFPGA, KERN_INFO "%s ..\n", __FUNCTION__); + + if (driver_registered) { + // Unregister this driver from the PCI bus driver + pci_unregister_driver(&pddf_multifpgapci_driver); + driver_registered = false; + } + + delete_all_fpga_data_nodes(); + KOBJ_FREE(multifpgapci_kobj) + + return; +} + +module_init(pddf_multifpgapci_driver_init); +module_exit(pddf_multifpgapci_driver_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Nexthop Systems"); +MODULE_DESCRIPTION("PDDF driver for systems with multiple PCI FPGAs"); diff --git a/platform/pddf/i2c/modules/multifpgapci/i2c/Makefile b/platform/pddf/i2c/modules/multifpgapci/i2c/Makefile new file mode 100644 index 00000000000..7e403df29c9 --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/i2c/Makefile @@ -0,0 +1,3 @@ +obj-m := pddf_multifpgapci_i2c_module.o + +ccflags-y := -I$(M)/modules/include \ No newline at end of file diff --git a/platform/pddf/i2c/modules/multifpgapci/i2c/pddf_multifpgapci_i2c_module.c b/platform/pddf/i2c/modules/multifpgapci/i2c/pddf_multifpgapci_i2c_module.c new file mode 100644 index 00000000000..b825693029f --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/i2c/pddf_multifpgapci_i2c_module.c @@ -0,0 +1,238 @@ +/* + * Copyright 2025 Nexthop Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Description: + * Platform MULTIFPGAPCI defines/structures header file + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "pddf_multifpgapci_defs.h" +#include "pddf_multifpgapci_i2c_defs.h" + +int (*pddf_i2c_multifpgapci_add_numbered_bus)(struct i2c_adapter *, int) = NULL; +EXPORT_SYMBOL(pddf_i2c_multifpgapci_add_numbered_bus); + +ssize_t new_i2c_adapter(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + int index, error; + + error = kstrtoint(buf, 10, &index); + if (error != 0) { + pddf_dbg(MULTIFPGA, KERN_ERR "Error converting string: %d\n", + error); + return -EINVAL; + } + + if (index < 0 || index >= I2C_PCI_MAX_BUS) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] I2C Adapter %d out of range [0, %d)\n", + __FUNCTION__, index, I2C_PCI_MAX_BUS); + return -ENODEV; + } + + struct pddf_data_attribute *_ptr = (struct pddf_data_attribute *)da; + struct pci_dev *pci_dev = (struct pci_dev *)_ptr->addr; + struct pddf_multifpgapci_drvdata *pci_drvdata = + pci_get_drvdata(pci_dev); + struct i2c_adapter_drvdata *i2c_privdata = + &pci_drvdata->i2c_adapter_drvdata; + struct i2c_adapter *i2c_adapters = i2c_privdata->i2c_adapters; + + if (i2c_privdata->i2c_adapter_registered[index]) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] I2C Adapter %d already exists\n", + __FUNCTION__, index); + return -ENODEV; + } + + i2c_adapters[index].owner = THIS_MODULE; + i2c_adapters[index].class = I2C_CLASS_HWMON | I2C_CLASS_SPD; + + // /dev/i2c-xxx for FPGA LOGIC I2C channel controller 1-7 + i2c_adapters[index].nr = index + i2c_privdata->virt_bus; + sprintf(i2c_adapters[index].name, "i2c-pci-%d", + index + i2c_privdata->virt_bus); + + // Set up the sysfs linkage to our parent device + i2c_adapters[index].dev.parent = &pci_dev->dev; + + if (pddf_i2c_multifpgapci_add_numbered_bus == NULL) { + pddf_dbg(MULTIFPGA, + KERN_ERR + "PDDF_I2C ERROR %s: MULTIFPGAPCIE add numbered bus " + "failed because fpga custom algo module is not loaded", + __func__); + return -ENODEV; + } + if ((pddf_i2c_multifpgapci_add_numbered_bus(&i2c_adapters[index], + index) != 0)) { + pddf_dbg(MULTIFPGA, + KERN_ERR "Cannot add bus %d to algorithm layer\n", + index); + return -ENODEV; + } + i2c_privdata->i2c_adapter_registered[index] = true; + pddf_dbg(MULTIFPGA, KERN_INFO "[%s] Registered bus id: %s\n", + __FUNCTION__, kobject_name(&i2c_adapters[index].dev.kobj)); + + return count; +} + +ssize_t del_i2c_adapter(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + int index, error; + + error = kstrtoint(buf, 10, &index); + if (error != 0) { + pddf_dbg(MULTIFPGA, KERN_ERR "Error converting string: %d\n", + error); + return -EINVAL; + } + + if (index < 0 || index >= I2C_PCI_MAX_BUS) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] I2C Adapter %d out of range [0, %d)\n", + __FUNCTION__, index, I2C_PCI_MAX_BUS); + return -ENODEV; + } + + struct pddf_data_attribute *_ptr = (struct pddf_data_attribute *)da; + struct i2c_adapter_drvdata *i2c_privdata = + (struct i2c_adapter_drvdata *)_ptr->addr; + struct i2c_adapter *i2c_adapters = i2c_privdata->i2c_adapters; + + if (!i2c_privdata->i2c_adapter_registered[index]) { + pddf_dbg(MULTIFPGA, + KERN_ERR "%s: I2C Adapter %d doesn't exist\n", + __FUNCTION__, index); + return -ENODEV; + } + + pddf_dbg(MULTIFPGA, + KERN_INFO "[%s] Attempting delete of bus index: %d\n", + __FUNCTION__, index); + + i2c_del_adapter(&i2c_adapters[index]); + + i2c_privdata->i2c_adapter_registered[index] = false; + + return count; +} + +int pddf_multifpgapci_i2c_module_init(struct pci_dev *pci_dev, + struct kobject *kobj) +{ + pddf_dbg(MULTIFPGA, KERN_INFO "[%s] pci_dev %s\n", __FUNCTION__, + pci_name(pci_dev)); + struct pddf_multifpgapci_drvdata *pci_drvdata = + pci_get_drvdata(pci_dev); + struct i2c_adapter_drvdata *i2c_privdata = + &pci_drvdata->i2c_adapter_drvdata; + int err; + + memset(i2c_privdata->i2c_adapter_registered, 0, + sizeof(i2c_privdata->i2c_adapter_registered)); + + PDDF_DATA_ATTR( + virt_bus, S_IWUSR | S_IRUGO, show_pddf_data, store_pddf_data, + PDDF_UINT32, sizeof(uint32_t), + (void *)&i2c_privdata->temp_sysfs_vals.virt_bus, NULL); + PDDF_DATA_ATTR( + ch_base_offset, S_IWUSR | S_IRUGO, show_pddf_data, + store_pddf_data, PDDF_UINT32, sizeof(uint32_t), + (void *)&i2c_privdata->temp_sysfs_vals.ch_base_offset, NULL); + PDDF_DATA_ATTR( + ch_size, S_IWUSR | S_IRUGO, show_pddf_data, store_pddf_data, + PDDF_UINT32, sizeof(uint32_t), + (void *)&i2c_privdata->temp_sysfs_vals.ch_size, NULL); + PDDF_DATA_ATTR( + num_virt_ch, S_IWUSR | S_IRUGO, show_pddf_data, store_pddf_data, + PDDF_UINT32, sizeof(uint32_t), + (void *)&i2c_privdata->temp_sysfs_vals.num_virt_ch, NULL); + PDDF_DATA_ATTR( + new_i2c_adapter, S_IWUSR | S_IRUGO, show_pddf_data, + new_i2c_adapter, PDDF_CHAR, NAME_SIZE, (void *)pci_dev, NULL); + PDDF_DATA_ATTR( + del_i2c_adapter, S_IWUSR | S_IRUGO, show_pddf_data, + del_i2c_adapter, PDDF_CHAR, NAME_SIZE, (void *)i2c_privdata, NULL); + + i2c_privdata->attrs.attr_virt_bus = attr_virt_bus; + i2c_privdata->attrs.attr_ch_base_offset = attr_ch_base_offset; + i2c_privdata->attrs.attr_ch_size = attr_ch_size; + i2c_privdata->attrs.attr_num_virt_ch = attr_num_virt_ch; + i2c_privdata->attrs.attr_new_i2c_adapter = attr_new_i2c_adapter; + i2c_privdata->attrs.attr_del_i2c_adapter = attr_del_i2c_adapter; + + // All the attributes are added in above step are put into here to be + // put in sysfs group + struct attribute *i2c_adapter_attrs[NUM_I2C_ADAPTER_ATTRS + 1] = { + &i2c_privdata->attrs.attr_virt_bus.dev_attr.attr, + &i2c_privdata->attrs.attr_ch_base_offset.dev_attr.attr, + &i2c_privdata->attrs.attr_ch_size.dev_attr.attr, + &i2c_privdata->attrs.attr_num_virt_ch.dev_attr.attr, + &i2c_privdata->attrs.attr_new_i2c_adapter.dev_attr.attr, + &i2c_privdata->attrs.attr_del_i2c_adapter.dev_attr.attr, + NULL, + }; + + memcpy(i2c_privdata->i2c_adapter_attrs, i2c_adapter_attrs, + sizeof(i2c_privdata->i2c_adapter_attrs)); + + i2c_privdata->i2c_adapter_attr_group.attrs = + i2c_privdata->i2c_adapter_attrs; + + err = sysfs_create_group(kobj, &i2c_privdata->i2c_adapter_attr_group); + if (err) { + pddf_dbg(MULTIFPGA, + KERN_ERR "[%s] sysfs_create_group error, status: %d\n", + __FUNCTION__, err); + return err; + } + + return 0; +} +EXPORT_SYMBOL(pddf_multifpgapci_i2c_module_init); + +void pddf_multifpgapci_i2c_module_exit(struct pci_dev *pci_dev, + struct kobject *kobj) +{ + struct pddf_multifpgapci_drvdata *pci_drvdata = + pci_get_drvdata(pci_dev); + struct i2c_adapter_drvdata *i2c_privdata = + &pci_drvdata->i2c_adapter_drvdata; + int i; + for (i = 0; i < I2C_PCI_MAX_BUS; i++) { + if (i2c_privdata->i2c_adapter_registered[i]) { + pddf_dbg(MULTIFPGA, + KERN_INFO "[%s] deleting i2c adapter: %s\n", + __FUNCTION__, + i2c_privdata->i2c_adapters[i].name); + i2c_del_adapter(&i2c_privdata->i2c_adapters[i]); + } + } + sysfs_remove_group(kobj, &i2c_privdata->i2c_adapter_attr_group); +}; +EXPORT_SYMBOL(pddf_multifpgapci_i2c_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Nexthop Systems"); +MODULE_DESCRIPTION("PDDF Platform Data for Multiple PCI FPGA I2C adapters."); \ No newline at end of file diff --git a/platform/pddf/i2c/modules/multifpgapci/pddf_multifpgapci_module.c b/platform/pddf/i2c/modules/multifpgapci/pddf_multifpgapci_module.c new file mode 100644 index 00000000000..3ffc758c594 --- /dev/null +++ b/platform/pddf/i2c/modules/multifpgapci/pddf_multifpgapci_module.c @@ -0,0 +1,194 @@ +/* + * Copyright 2025 Nexthop Systems Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * A PDDF kernel module to create sysfs for multiple PCI FPGAs. + * Borrows from the PDDF fpgapci module by Broadcom. + */ + +#include +#include +#include +#include +#include +#include + +#include "pddf_client_defs.h" +#include "pddf_multifpgapci_defs.h" + +#define MAX_PCI_IDS 16 + +static struct kobject *multifpgapci_kobj = NULL; +struct pci_device_id fpgapci_ids[MAX_PCI_IDS + 1]; +static int num_pci_ids = 0; + +extern int pddf_multifpgapci_register(const struct pci_device_id *ids, + struct kobject *kobj); + +static ssize_t dev_operation(struct device *dev, struct device_attribute *da, + const char *buf, size_t count); +static ssize_t register_pci_device_id(struct device *dev, + struct device_attribute *da, + const char *buf, size_t count); + +PDDF_DATA_ATTR(dev_ops, S_IWUSR | S_IRUGO, show_pddf_data, dev_operation, + PDDF_CHAR, NAME_SIZE, NULL, NULL); +PDDF_DATA_ATTR(register_pci_device_id, S_IWUSR | S_IRUGO, show_pddf_data, + register_pci_device_id, PDDF_CHAR, NAME_SIZE, NULL, NULL); + +struct attribute *attrs_multifpgapci[] = { + &attr_dev_ops.dev_attr.attr, + &attr_register_pci_device_id.dev_attr.attr, + NULL, +}; + +struct attribute_group attr_group_multifpgapci = { + .attrs = attrs_multifpgapci, +}; + +ssize_t dev_operation(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + if (strncmp(buf, "multifpgapci_init", strlen("multifpgapci_init")) == + 0) { + if (num_pci_ids > 0) { + pddf_multifpgapci_register(fpgapci_ids, + multifpgapci_kobj); + } else { + pddf_dbg( + MULTIFPGA, + KERN_ERR + "PDDF_ERROR %s: No FPGA PCI IDs are registered yet\n", + __FUNCTION__); + return -EINVAL; + } + } else { + pddf_dbg(MULTIFPGA, + KERN_ERR + "PDDF_ERROR %s: Invalid value for dev_ops %s\n", + __FUNCTION__, buf); + return -EINVAL; + } + + return count; +} + +int add_fpgapci_id(unsigned short vendor, unsigned short device) +{ + if (num_pci_ids < MAX_PCI_IDS) { + fpgapci_ids[num_pci_ids].vendor = vendor; + fpgapci_ids[num_pci_ids].device = device; + fpgapci_ids[num_pci_ids].subvendor = PCI_ANY_ID; + fpgapci_ids[num_pci_ids].subdevice = PCI_ANY_ID; + fpgapci_ids[num_pci_ids].class = PCI_ANY_ID; + fpgapci_ids[num_pci_ids].class_mask = 0; + fpgapci_ids[num_pci_ids].driver_data = 0; + num_pci_ids++; + // Null-terminate the array + fpgapci_ids[num_pci_ids] = (struct pci_device_id){ 0 }; + pddf_dbg(MULTIFPGA, + KERN_INFO + "%s Registered vendor: 0x%04x, device: 0x%04x\n", + __FUNCTION__, vendor, device); + return 0; + } else { + pddf_dbg( + MULTIFPGA, + KERN_ERR + "PDDF_ERROR %s: Maximum number of FPGA PCI IDs reached\n", + __FUNCTION__); + return -1; + } +} + +ssize_t register_pci_device_id(struct device *dev, struct device_attribute *da, + const char *buf, size_t count) +{ + unsigned int vendor, device; + int sscanf_result; + + // We expect the vendor and device ID in hex format + // Try parsing with "0x" prefix first + sscanf_result = sscanf(buf, "0x%x 0x%x", &vendor, &device); + if (sscanf_result == 2) { + if (add_fpgapci_id((unsigned short)vendor, + (unsigned short)device) == 0) { + return count; + } + return -ENOSPC; + } + + // Try parsing without "0x" prefix + sscanf_result = sscanf(buf, "%x %x", &vendor, &device); + if (sscanf_result == 2) { + if (add_fpgapci_id((unsigned short)vendor, + (unsigned short)device) == 0) { + return count; + } + return -ENOSPC; + } + + pddf_dbg(MULTIFPGA, + KERN_ERR + "%s Failed to register pci device ids, unexpected format\n", + __FUNCTION__); + return -EINVAL; +} + +int __init pddf_multifpgapci_module_init(void) +{ + int ret = 0; + struct kobject *device_kobj; + + pddf_dbg(MULTIFPGA, KERN_INFO "%s ..\n", __FUNCTION__); + + device_kobj = get_device_i2c_kobj(); + if (!device_kobj) { + pddf_dbg(MULTIFPGA, + KERN_ERR "%s get_device_i2c_kobj failed ..\n", + __FUNCTION__); + return -ENOMEM; + } + + multifpgapci_kobj = kobject_create_and_add("multifpgapci", device_kobj); + if (!multifpgapci_kobj) { + pddf_dbg(MULTIFPGA, + KERN_ERR "%s create multifpgapci kobj failed ..\n", + __FUNCTION__); + return -ENOMEM; + } + + ret = sysfs_create_group(multifpgapci_kobj, &attr_group_multifpgapci); + if (ret) { + pddf_dbg(MULTIFPGA, + KERN_ERR + "%s create multifpgapci sysfs attributes failed ..\n", + __FUNCTION__); + return ret; + } + + return 0; +} + +void __exit pddf_multifpgapci_module_exit(void) +{ + pddf_dbg(MULTIFPGA, KERN_INFO "%s ..\n", __FUNCTION__); + KOBJ_FREE(multifpgapci_kobj) + return; +} + +module_init(pddf_multifpgapci_module_init); +module_exit(pddf_multifpgapci_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Nexthop Systems"); +MODULE_DESCRIPTION("PDDF module for systems with multiple PCI FPGAs"); diff --git a/platform/pddf/i2c/utils/pddfparse.py b/platform/pddf/i2c/utils/pddfparse.py index 9357e53ff95..d8ef380b118 100755 --- a/platform/pddf/i2c/utils/pddfparse.py +++ b/platform/pddf/i2c/utils/pddfparse.py @@ -467,6 +467,54 @@ def create_fpgapci_device(self, dev, ops): ret = self.runcmd(cmd) return create_ret.append(ret) + def create_multifpgapcisystem_device(self, dev, ops): + create_ret = [] + ret = 0 + for i in dev['dev_attr']['PCI_DEVICE_IDS']: + cmd = "echo '{} {}' > /sys/kernel/pddf/devices/multifpgapci/register_pci_device_id".format(i['vendor'], i['device']) + ret = self.runcmd(cmd) + if ret != 0: + return create_ret.append(ret) + + cmd = "echo 'multifpgapci_init' > /sys/kernel/pddf/devices/multifpgapci/dev_ops" + ret = self.runcmd(cmd) + return create_ret.append(ret) + + def create_multifpgapci_device(self, dev, ops): + create_ret = [] + ret = 0 + bdf = dev['dev_info']['device_bdf'] + + # Top level data store + ret = self.create_device(dev['dev_info']['dev_attr'], "pddf/devices/multifpgapci/{}".format(bdf), ops) + if ret != 0: + return create_ret.append(ret) + + # PDDF client data store + cmd = "echo '{}' > /sys/kernel/pddf/devices/multifpgapci/{}/i2c_name".format(dev['dev_info']['device_name'], bdf) + ret = self.runcmd(cmd) + if ret != 0: + return create_ret.append(ret) + + # I2C specific data store + ret = self.create_device(dev['i2c']['dev_attr'], "pddf/devices/multifpgapci/{}/i2c".format(bdf), ops) + if ret != 0: + return create_ret.append(ret) + + # TODO: add GPIO & SPI specific data stores + + cmd = "echo 'fpgapci_init' > /sys/kernel/pddf/devices/multifpgapci/{}/dev_ops".format(bdf) + ret = self.runcmd(cmd) + if ret != 0: + return create_ret.append(ret) + + for bus in range(int(dev['i2c']['dev_attr']['num_virt_ch'], 16)): + cmd = "echo {} > /sys/kernel/pddf/devices/multifpgapci/{}/i2c/new_i2c_adapter".format(bus, bdf) + ret = self.runcmd(cmd) + if ret != 0: + return create_ret.append(ret) + + return create_ret.append(ret) ################################################################################################################################# # DELETE DEFS @@ -587,6 +635,15 @@ def delete_psu_device(self, dev, ops): def delete_fpgapci_device(self, dev, ops): return + + def delete_multifpgapci_device(self, dev, ops): + bdf = dev['dev_info']['device_bdf'] + + cmd = "echo '{}' > /sys/kernel/pddf/devices/multifpgapci/{}/i2c_name".format(dev['dev_info']['device_name'], bdf) + self.runcmd(cmd) + cmd = "echo 'fpgapci_deinit' > /sys/kernel/pddf/devices/multifpgapci/{}/dev_ops".format(bdf) + self.runcmd(cmd) + ################################################################################################################################# # SHOW ATTRIBIUTES DEFS ################################################################################################################### @@ -872,6 +929,14 @@ def show_attr_fpgapci_device(self, dev, ops): return ret + def show_attr_multifpgapci_device(self, dev, ops): + ret = [] + KEY="multifpgapci" + if KEY not in self.data_sysfs_obj: + self.data_sysfs_obj[KEY] = [] + + return ret + ################################################################################################################### # SHOW DEFS @@ -1105,6 +1170,9 @@ def show_fpgapci_device(self, dev, ops): '/sys/kernel/pddf/devices/fpgapci/virt_i2c_ch'] self.add_list_sysfs_obj(self.sysfs_obj, KEY, extra_list) + def show_multifpgapci_device(self, dev, ops): + # TODO: Implement this once we've finalized the sysfs interface (i.e. i2c, spi directories) + return def validate_xcvr_device(self, dev, ops): devtype_list = ['optoe1', 'optoe2'] @@ -1665,6 +1733,53 @@ def fpgapci_parse(self, dev, ops): val.extend(ret) return val + def multifpgapcisystem_parse(self, ops): + val = [] + + dev = self.data.get("MULTIFPGAPCIESYSTEM0") + if not dev: + return [] + + if "dev_info" not in dev: + return [] + + if dev["dev_info"].get("device_type") != "MULTIFPGAPCIESYSTEM": + return [] + + ret = getattr(self, ops['cmd']+"_multifpgapcisystem_device")(dev, ops) + if ret: + if str(ret[0]).isdigit(): + if ret[0] != 0: + print("{}_multifpgapcisystem_device() cmd failed".format(ops['cmd'])) + return ret + else: + val.extend(ret) + + return val + + def multifpgapci_parse(self, dev, ops): + val = [] + ret = getattr(self, ops['cmd']+"_multifpgapci_device")(dev, ops) + if ret: + if str(ret[0]).isdigit(): + if ret[0] != 0: + # in case if 'create' functions + print("{}_multifpgapci_device() cmd failed".format(ops['cmd'])) + return ret + else: + val.extend(ret) + + for bus in dev['i2c']['channel']: + ret = self.dev_parse(self.data[bus['dev']], ops) + if ret: + if str(ret[0]).isdigit(): + if ret[0] != 0: + # in case if 'create' functions + return ret + else: + val.extend(ret) + return val + # 'create' and 'show_attr' ops returns an array # 'delete', 'show' and 'validate' ops return None def dev_parse(self, dev, ops): @@ -1719,6 +1834,9 @@ def dev_parse(self, dev, ops): if attr['device_type'] == 'SYSSTAT': return self.sysstatus_parse(dev, ops) + if attr['device_type'] == 'MULTIFPGAPCIE': + return self.multifpgapci_parse(dev, ops) + if attr['device_type'] == 'DCDC': return self.dcdc_parse(dev, ops) @@ -1759,10 +1877,9 @@ def create_led_device(self, key, ops): if 'attr_devtype' not in attr.keys(): self.create_attr('attr_devtype', 'cpld', path) for attr_key in attr.keys(): - if (attr_key == 'swpld_addr_offset' or attr_key == 'swpld_addr' or \ - attr_key == 'attr_devtype' or attr_key == 'attr_devname' ): + if attr_key in ['swpld_addr_offset', 'swpld_addr', 'attr_devtype', 'attr_devname', 'attr_bdf']: self.create_attr(attr_key, attr[attr_key], path) - elif (attr_key != 'attr_name' and attr_key != 'descr' and attr_key != 'state'): + elif attr_key not in ['attr_name', 'descr', 'state']: state_path = path+'/state_attr' self.create_attr(attr_key, attr[attr_key],state_path) cmd="echo '" + attr['attr_name']+"' > /sys/kernel/pddf/devices/led/dev_ops" @@ -1788,6 +1905,10 @@ def get_device_list(self, list, type): def create_pddf_devices(self): + ret = self.multifpgapcisystem_parse({"cmd": "create", "target": "all", "attr": "all"}) + if ret: + if ret[0] != 0: + return ret[0] self.led_parse({"cmd": "create", "target": "all", "attr": "all"}) create_ret = 0 ret = self.dev_parse(self.data['SYSTEM'], {"cmd": "create", "target": "all", "attr": "all"}) diff --git a/platform/pddf/platform-api-pddf-base/sonic_platform_pddf_base/pddfapi.py b/platform/pddf/platform-api-pddf-base/sonic_platform_pddf_base/pddfapi.py index adff58df4c7..4809d6cf1ba 100644 --- a/platform/pddf/platform-api-pddf-base/sonic_platform_pddf_base/pddfapi.py +++ b/platform/pddf/platform-api-pddf-base/sonic_platform_pddf_base/pddfapi.py @@ -523,6 +523,14 @@ def show_attr_cpld_device(self, dev, ops): return ret + def show_attr_multifpgapci_device(self, dev, ops): + ret = [] + KEY="multifpgapci" + if KEY not in self.data_sysfs_obj: + self.data_sysfs_obj[KEY] = [] + + return ret + ################################################################################################################### # SPYTEST ################################################################################################################### @@ -817,6 +825,29 @@ def cpu_parse_reverse(self, bus, ops): val.extend(ret) return val + def multifpgapci_parse(self, dev, ops): + val = [] + ret = getattr(self, ops['cmd']+"_multifpgapci_device")(dev, ops) + if ret: + if str(ret[0]).isdigit(): + if ret[0] != 0: + # in case if 'create' functions + print("{}_multifpgapci_device() cmd failed".format(ops['cmd'])) + return ret + else: + val.extend(ret) + + for bus in dev['i2c']['channel']: + ret = self.dev_parse(self.data[bus['dev']], ops) + if ret: + if str(ret[0]).isdigit(): + if ret[0] != 0: + # in case if 'create' functions + return ret + else: + val.extend(ret) + return val + def dev_parse(self, dev, ops): attr = dev['dev_info'] if attr['device_type'] == 'CPU': @@ -864,6 +895,9 @@ def dev_parse(self, dev, ops): if attr['device_type'] == 'SYSSTAT': return self.sysstatus_parse(dev, ops) + if attr['device_type'] == 'MULTIFPGAPCIE': + return self.multifpgapci_parse(dev, ops) + def create_attr(self, key, value, path): cmd = "echo '%s' > /sys/kernel/%s/%s" % (value, path, key) self.runcmd(cmd)