/* SPDX-License-Identifier: AGPL-3.0-or-later */ /* Copyright (C) 2026 David (Corey) Herrera */ /* * bar1_size_probe.c — Probe GK210 BAR1 minimum size * * Acquires a die via bar_swap, disables decode, writes 0xFFFFFFFF to BAR1 * config registers, reads back the size mask, restores original values. * This reveals which bits are hardwired (minimum BAR size). * * Must run as root. bar_swap.ko must be loaded. nvidia must NOT be loaded. */ #include #include #include #include #include #include #include #include #include "../bar_swap_ko/bar_swap.h" /* Find GK210 sysfs path by scanning PCI devices */ static int find_gk210_bdf(int die_id, char *bdf_out, size_t bdf_sz) { /* Just use setpci on known GK210 device IDs */ /* For now, scan /sys/bus/pci/devices/ for vendor=10de device=102d */ DIR *dir = opendir("/sys/bus/pci/devices"); if (!dir) return -1; int found = 0; struct dirent *ent; while ((ent = readdir(dir)) != NULL) { if (ent->d_name[0] == '.') continue; char path[512]; snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/vendor", ent->d_name); FILE *f = fopen(path, "r"); if (!f) continue; unsigned vendor; fscanf(f, "%x", &vendor); fclose(f); if (vendor != 0x10de) continue; snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/device", ent->d_name); f = fopen(path, "r"); if (!f) continue; unsigned device; fscanf(f, "%x", &device); fclose(f); if (device != 0x102d) continue; if (found == die_id) { snprintf(bdf_out, bdf_sz, "%s", ent->d_name); closedir(dir); return 0; } found++; } closedir(dir); return -1; } static int read_pci_config(const char *bdf, int offset, uint32_t *val) { char path[512]; snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config", bdf); int fd = open(path, O_RDONLY); if (fd < 0) { perror(path); return -1; } if (lseek(fd, offset, SEEK_SET) < 0) { perror("lseek"); close(fd); return -1; } if (read(fd, val, 4) != 4) { perror("read"); close(fd); return -1; } close(fd); return 0; } static int write_pci_config(const char *bdf, int offset, uint32_t val) { char path[512]; snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config", bdf); int fd = open(path, O_RDWR); if (fd < 0) { perror(path); return -1; } if (lseek(fd, offset, SEEK_SET) < 0) { perror("lseek"); close(fd); return -1; } if (write(fd, &val, 4) != 4) { perror("write"); close(fd); return -1; } close(fd); return 0; } int main(int argc, char **argv) { int die_id = 0; if (argc > 1) die_id = atoi(argv[1]); printf("=== GK210 BAR1 Size Probe ===\n\n"); /* Open bar_swap */ int bs_fd = open("/dev/bar_swap", O_RDWR); if (bs_fd < 0) { perror("/dev/bar_swap"); return 1; } /* Query */ struct bs_query_info qi; if (ioctl(bs_fd, BAR_SWAP_QUERY, &qi) < 0) { perror("QUERY"); close(bs_fd); return 1; } printf("Dies: %u, Active: %s\n", qi.n_dies, qi.active_die == 0xFFFFFFFF ? "none" : "occupied"); if ((unsigned)die_id >= qi.n_dies) { printf("Die %d out of range (max %u)\n", die_id, qi.n_dies - 1); close(bs_fd); return 1; } /* Find GK210 BDF */ char bdf[64]; if (find_gk210_bdf(die_id, bdf, sizeof(bdf)) < 0) { printf("Could not find GK210 die %d in sysfs\n", die_id); close(bs_fd); return 1; } printf("Die %d: %s\n\n", die_id, bdf); /* Acquire die */ struct bs_acquire_req aq = { .die_id = die_id, .flags = 0 }; if (ioctl(bs_fd, BAR_SWAP_ACQUIRE, &aq) < 0) { perror("ACQUIRE"); close(bs_fd); return 1; } printf("Die %d acquired. BAR0=0x%llx BAR1=0x%llx BAR3=0x%llx\n\n", die_id, aq.bar0_addr, aq.bar1_addr, aq.bar3_addr); /* * BAR1 is a 64-bit BAR at PCI config offset 0x14 (BAR2 in PCI terms, * since BAR0 is at 0x10 and is 64-bit, taking offsets 0x10+0x14). * * Actually for GK210: * BAR0 = offset 0x10 (32-bit, 16 MB) * BAR1 = offset 0x14 + 0x18 (64-bit, 16 GB) * BAR3 = offset 0x1C + 0x20 (64-bit, 32 MB) * * Let's just read all 6 BAR registers first. */ printf("Current BAR config registers:\n"); uint32_t bars[6]; for (int i = 0; i < 6; i++) { if (read_pci_config(bdf, 0x10 + i * 4, &bars[i]) < 0) { printf(" Failed to read BAR%d\n", i); goto release; } printf(" BAR%d (0x%02X): 0x%08X\n", i, 0x10 + i * 4, bars[i]); } /* Identify BAR1 (the 16 GB VRAM BAR) by checking which is 64-bit prefetchable */ printf("\nBAR type analysis:\n"); for (int i = 0; i < 6; i++) { uint32_t v = bars[i]; if (v & 1) { printf(" BAR%d: I/O\n", i); } else { int type = (v >> 1) & 3; /* 0=32bit, 2=64bit */ int pref = (v >> 3) & 1; printf(" BAR%d: Memory, %s, %s\n", i, type == 2 ? "64-bit" : "32-bit", pref ? "prefetchable" : "non-prefetchable"); if (type == 2) { printf(" (high dword in BAR%d: 0x%08X)\n", i + 1, bars[i + 1]); i++; /* skip high dword */ } } } /* * Now probe BAR1 size. * For GK210: BAR0 at 0x10 is 32-bit non-pref (registers). * BAR1 at 0x14+0x18 is 64-bit pref (VRAM). * * To probe size: * 1. Disable memory decode (clear CMD bit 1) * 2. Save BAR1 low + high * 3. Write 0xFFFFFFFF to BAR1 low, read back * 4. Write 0xFFFFFFFF to BAR1 high, read back * 5. Restore original values * 6. Re-enable memory decode */ /* BAR1 is at offset 0x14 (low) and 0x18 (high) based on GK210 layout */ int bar1_lo_off = -1, bar1_hi_off = -1; /* Find the 64-bit prefetchable BAR */ for (int i = 0; i < 6; i++) { uint32_t v = bars[i]; if (!(v & 1) && ((v >> 1) & 3) == 2 && ((v >> 3) & 1)) { /* 64-bit prefetchable memory BAR */ bar1_lo_off = 0x10 + i * 4; bar1_hi_off = 0x10 + (i + 1) * 4; printf("\nBAR1 (VRAM) found at config offset 0x%02X/0x%02X\n", bar1_lo_off, bar1_hi_off); break; } } if (bar1_lo_off < 0) { printf("\nERROR: Could not find 64-bit prefetchable BAR\n"); goto release; } /* Save originals */ uint32_t orig_lo = bars[(bar1_lo_off - 0x10) / 4]; uint32_t orig_hi = bars[(bar1_hi_off - 0x10) / 4]; printf("Original BAR1: low=0x%08X high=0x%08X\n", orig_lo, orig_hi); printf("Current BAR1 address: 0x%llX\n", ((uint64_t)orig_hi << 32) | (orig_lo & 0xFFFFFFF0)); /* Disable decode */ uint32_t cmd; read_pci_config(bdf, 0x04, &cmd); uint32_t cmd_disabled = cmd & ~0x06; /* clear MEM + BusMaster */ write_pci_config(bdf, 0x04, cmd_disabled); printf("\nDecode disabled (CMD: 0x%04X -> 0x%04X)\n", cmd, cmd_disabled); /* Write all-ones to BAR1 low */ write_pci_config(bdf, bar1_lo_off, 0xFFFFFFFF); uint32_t mask_lo; read_pci_config(bdf, bar1_lo_off, &mask_lo); /* Write all-ones to BAR1 high */ write_pci_config(bdf, bar1_hi_off, 0xFFFFFFFF); uint32_t mask_hi; read_pci_config(bdf, bar1_hi_off, &mask_hi); printf("\n=== SIZE MASK RESULTS ===\n"); printf("BAR1 low mask: 0x%08X\n", mask_lo); printf("BAR1 high mask: 0x%08X\n", mask_hi); /* Decode the size */ /* For a memory BAR, bits [3:0] are type/pref flags, rest is size mask */ uint64_t full_mask = ((uint64_t)mask_hi << 32) | (mask_lo & 0xFFFFFFF0); uint64_t size = ~full_mask + 1; /* Two's complement gives size */ printf("\nFull mask (excluding type bits): 0x%016llX\n", (unsigned long long)full_mask); printf("Decoded BAR1 size: 0x%llX (%llu GB)\n", (unsigned long long)size, (unsigned long long)(size >> 30)); if (mask_lo == 0xFFFFFFFF && mask_hi == 0xFFFFFFFF) { printf("\nWARNING: BAR1 returned all-ones — may not be accessible\n"); } /* Check if smaller sizes are possible */ printf("\n=== ANALYSIS ===\n"); if (size == 0x400000000ULL) { printf("BAR1 is hardwired to 16 GB. Cannot resize.\n"); printf("Multi-slot via BAR1 resizing is NOT possible.\n"); printf("Fallback: NIC relocation to free 0x400000000 slot.\n"); } else if (size < 0x400000000ULL) { printf("BAR1 minimum size: %llu GB — SMALLER THAN 16 GB!\n", (unsigned long long)(size >> 30)); printf("Multi-slot via BAR1 resizing IS possible!\n"); printf("Could fit %llu dies in the 48 GB MMIO window.\n", 48ULL / (size >> 30)); } else { printf("BAR1 size %llu GB — larger than expected\n", (unsigned long long)(size >> 30)); } /* Restore original BAR1 values */ write_pci_config(bdf, bar1_lo_off, orig_lo); write_pci_config(bdf, bar1_hi_off, orig_hi); /* Re-enable decode */ write_pci_config(bdf, 0x04, cmd); printf("\nBAR1 restored, decode re-enabled.\n"); release: /* Release die */ { struct bs_release_req rr = { .die_id = die_id, .flags = 0 }; ioctl(bs_fd, BAR_SWAP_RELEASE, &rr); printf("Die %d released.\n", die_id); } close(bs_fd); return 0; }