272 lines
9.3 KiB
C
272 lines
9.3 KiB
C
/* SPDX-License-Identifier: AGPL-3.0-or-later */
|
|
/* Copyright (C) 2026 David (Corey) Herrera */
|
|
/*
|
|
* bar1_size_probe.c — Probe GK210 BAR1 minimum size
|
|
*
|
|
* Acquires a die via bar_swap, disables decode, writes 0xFFFFFFFF to BAR1
|
|
* config registers, reads back the size mask, restores original values.
|
|
* This reveals which bits are hardwired (minimum BAR size).
|
|
*
|
|
* Must run as root. bar_swap.ko must be loaded. nvidia must NOT be loaded.
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
#include <fcntl.h>
|
|
#include <unistd.h>
|
|
#include <dirent.h>
|
|
#include <sys/ioctl.h>
|
|
|
|
#include "../bar_swap_ko/bar_swap.h"
|
|
|
|
/* Find GK210 sysfs path by scanning PCI devices */
|
|
static int find_gk210_bdf(int die_id, char *bdf_out, size_t bdf_sz) {
|
|
/* Just use setpci on known GK210 device IDs */
|
|
/* For now, scan /sys/bus/pci/devices/ for vendor=10de device=102d */
|
|
DIR *dir = opendir("/sys/bus/pci/devices");
|
|
if (!dir) return -1;
|
|
|
|
int found = 0;
|
|
struct dirent *ent;
|
|
while ((ent = readdir(dir)) != NULL) {
|
|
if (ent->d_name[0] == '.') continue;
|
|
|
|
char path[512];
|
|
snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/vendor", ent->d_name);
|
|
FILE *f = fopen(path, "r");
|
|
if (!f) continue;
|
|
unsigned vendor;
|
|
fscanf(f, "%x", &vendor);
|
|
fclose(f);
|
|
if (vendor != 0x10de) continue;
|
|
|
|
snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/device", ent->d_name);
|
|
f = fopen(path, "r");
|
|
if (!f) continue;
|
|
unsigned device;
|
|
fscanf(f, "%x", &device);
|
|
fclose(f);
|
|
if (device != 0x102d) continue;
|
|
|
|
if (found == die_id) {
|
|
snprintf(bdf_out, bdf_sz, "%s", ent->d_name);
|
|
closedir(dir);
|
|
return 0;
|
|
}
|
|
found++;
|
|
}
|
|
closedir(dir);
|
|
return -1;
|
|
}
|
|
|
|
static int read_pci_config(const char *bdf, int offset, uint32_t *val) {
|
|
char path[512];
|
|
snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config", bdf);
|
|
int fd = open(path, O_RDONLY);
|
|
if (fd < 0) { perror(path); return -1; }
|
|
if (lseek(fd, offset, SEEK_SET) < 0) { perror("lseek"); close(fd); return -1; }
|
|
if (read(fd, val, 4) != 4) { perror("read"); close(fd); return -1; }
|
|
close(fd);
|
|
return 0;
|
|
}
|
|
|
|
static int write_pci_config(const char *bdf, int offset, uint32_t val) {
|
|
char path[512];
|
|
snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/config", bdf);
|
|
int fd = open(path, O_RDWR);
|
|
if (fd < 0) { perror(path); return -1; }
|
|
if (lseek(fd, offset, SEEK_SET) < 0) { perror("lseek"); close(fd); return -1; }
|
|
if (write(fd, &val, 4) != 4) { perror("write"); close(fd); return -1; }
|
|
close(fd);
|
|
return 0;
|
|
}
|
|
|
|
int main(int argc, char **argv) {
|
|
int die_id = 0;
|
|
if (argc > 1) die_id = atoi(argv[1]);
|
|
|
|
printf("=== GK210 BAR1 Size Probe ===\n\n");
|
|
|
|
/* Open bar_swap */
|
|
int bs_fd = open("/dev/bar_swap", O_RDWR);
|
|
if (bs_fd < 0) { perror("/dev/bar_swap"); return 1; }
|
|
|
|
/* Query */
|
|
struct bs_query_info qi;
|
|
if (ioctl(bs_fd, BAR_SWAP_QUERY, &qi) < 0) { perror("QUERY"); close(bs_fd); return 1; }
|
|
printf("Dies: %u, Active: %s\n", qi.n_dies, qi.active_die == 0xFFFFFFFF ? "none" : "occupied");
|
|
|
|
if ((unsigned)die_id >= qi.n_dies) {
|
|
printf("Die %d out of range (max %u)\n", die_id, qi.n_dies - 1);
|
|
close(bs_fd);
|
|
return 1;
|
|
}
|
|
|
|
/* Find GK210 BDF */
|
|
char bdf[64];
|
|
if (find_gk210_bdf(die_id, bdf, sizeof(bdf)) < 0) {
|
|
printf("Could not find GK210 die %d in sysfs\n", die_id);
|
|
close(bs_fd);
|
|
return 1;
|
|
}
|
|
printf("Die %d: %s\n\n", die_id, bdf);
|
|
|
|
/* Acquire die */
|
|
struct bs_acquire_req aq = { .die_id = die_id, .flags = 0 };
|
|
if (ioctl(bs_fd, BAR_SWAP_ACQUIRE, &aq) < 0) { perror("ACQUIRE"); close(bs_fd); return 1; }
|
|
printf("Die %d acquired. BAR0=0x%llx BAR1=0x%llx BAR3=0x%llx\n\n",
|
|
die_id, aq.bar0_addr, aq.bar1_addr, aq.bar3_addr);
|
|
|
|
/*
|
|
* BAR1 is a 64-bit BAR at PCI config offset 0x14 (BAR2 in PCI terms,
|
|
* since BAR0 is at 0x10 and is 64-bit, taking offsets 0x10+0x14).
|
|
*
|
|
* Actually for GK210:
|
|
* BAR0 = offset 0x10 (32-bit, 16 MB)
|
|
* BAR1 = offset 0x14 + 0x18 (64-bit, 16 GB)
|
|
* BAR3 = offset 0x1C + 0x20 (64-bit, 32 MB)
|
|
*
|
|
* Let's just read all 6 BAR registers first.
|
|
*/
|
|
printf("Current BAR config registers:\n");
|
|
uint32_t bars[6];
|
|
for (int i = 0; i < 6; i++) {
|
|
if (read_pci_config(bdf, 0x10 + i * 4, &bars[i]) < 0) {
|
|
printf(" Failed to read BAR%d\n", i);
|
|
goto release;
|
|
}
|
|
printf(" BAR%d (0x%02X): 0x%08X\n", i, 0x10 + i * 4, bars[i]);
|
|
}
|
|
|
|
/* Identify BAR1 (the 16 GB VRAM BAR) by checking which is 64-bit prefetchable */
|
|
printf("\nBAR type analysis:\n");
|
|
for (int i = 0; i < 6; i++) {
|
|
uint32_t v = bars[i];
|
|
if (v & 1) {
|
|
printf(" BAR%d: I/O\n", i);
|
|
} else {
|
|
int type = (v >> 1) & 3; /* 0=32bit, 2=64bit */
|
|
int pref = (v >> 3) & 1;
|
|
printf(" BAR%d: Memory, %s, %s\n", i,
|
|
type == 2 ? "64-bit" : "32-bit",
|
|
pref ? "prefetchable" : "non-prefetchable");
|
|
if (type == 2) {
|
|
printf(" (high dword in BAR%d: 0x%08X)\n", i + 1, bars[i + 1]);
|
|
i++; /* skip high dword */
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Now probe BAR1 size.
|
|
* For GK210: BAR0 at 0x10 is 32-bit non-pref (registers).
|
|
* BAR1 at 0x14+0x18 is 64-bit pref (VRAM).
|
|
*
|
|
* To probe size:
|
|
* 1. Disable memory decode (clear CMD bit 1)
|
|
* 2. Save BAR1 low + high
|
|
* 3. Write 0xFFFFFFFF to BAR1 low, read back
|
|
* 4. Write 0xFFFFFFFF to BAR1 high, read back
|
|
* 5. Restore original values
|
|
* 6. Re-enable memory decode
|
|
*/
|
|
|
|
/* BAR1 is at offset 0x14 (low) and 0x18 (high) based on GK210 layout */
|
|
int bar1_lo_off = -1, bar1_hi_off = -1;
|
|
|
|
/* Find the 64-bit prefetchable BAR */
|
|
for (int i = 0; i < 6; i++) {
|
|
uint32_t v = bars[i];
|
|
if (!(v & 1) && ((v >> 1) & 3) == 2 && ((v >> 3) & 1)) {
|
|
/* 64-bit prefetchable memory BAR */
|
|
bar1_lo_off = 0x10 + i * 4;
|
|
bar1_hi_off = 0x10 + (i + 1) * 4;
|
|
printf("\nBAR1 (VRAM) found at config offset 0x%02X/0x%02X\n", bar1_lo_off, bar1_hi_off);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (bar1_lo_off < 0) {
|
|
printf("\nERROR: Could not find 64-bit prefetchable BAR\n");
|
|
goto release;
|
|
}
|
|
|
|
/* Save originals */
|
|
uint32_t orig_lo = bars[(bar1_lo_off - 0x10) / 4];
|
|
uint32_t orig_hi = bars[(bar1_hi_off - 0x10) / 4];
|
|
printf("Original BAR1: low=0x%08X high=0x%08X\n", orig_lo, orig_hi);
|
|
printf("Current BAR1 address: 0x%llX\n",
|
|
((uint64_t)orig_hi << 32) | (orig_lo & 0xFFFFFFF0));
|
|
|
|
/* Disable decode */
|
|
uint32_t cmd;
|
|
read_pci_config(bdf, 0x04, &cmd);
|
|
uint32_t cmd_disabled = cmd & ~0x06; /* clear MEM + BusMaster */
|
|
write_pci_config(bdf, 0x04, cmd_disabled);
|
|
printf("\nDecode disabled (CMD: 0x%04X -> 0x%04X)\n", cmd, cmd_disabled);
|
|
|
|
/* Write all-ones to BAR1 low */
|
|
write_pci_config(bdf, bar1_lo_off, 0xFFFFFFFF);
|
|
uint32_t mask_lo;
|
|
read_pci_config(bdf, bar1_lo_off, &mask_lo);
|
|
|
|
/* Write all-ones to BAR1 high */
|
|
write_pci_config(bdf, bar1_hi_off, 0xFFFFFFFF);
|
|
uint32_t mask_hi;
|
|
read_pci_config(bdf, bar1_hi_off, &mask_hi);
|
|
|
|
printf("\n=== SIZE MASK RESULTS ===\n");
|
|
printf("BAR1 low mask: 0x%08X\n", mask_lo);
|
|
printf("BAR1 high mask: 0x%08X\n", mask_hi);
|
|
|
|
/* Decode the size */
|
|
/* For a memory BAR, bits [3:0] are type/pref flags, rest is size mask */
|
|
uint64_t full_mask = ((uint64_t)mask_hi << 32) | (mask_lo & 0xFFFFFFF0);
|
|
uint64_t size = ~full_mask + 1; /* Two's complement gives size */
|
|
|
|
printf("\nFull mask (excluding type bits): 0x%016llX\n", (unsigned long long)full_mask);
|
|
printf("Decoded BAR1 size: 0x%llX (%llu GB)\n",
|
|
(unsigned long long)size, (unsigned long long)(size >> 30));
|
|
|
|
if (mask_lo == 0xFFFFFFFF && mask_hi == 0xFFFFFFFF) {
|
|
printf("\nWARNING: BAR1 returned all-ones — may not be accessible\n");
|
|
}
|
|
|
|
/* Check if smaller sizes are possible */
|
|
printf("\n=== ANALYSIS ===\n");
|
|
if (size == 0x400000000ULL) {
|
|
printf("BAR1 is hardwired to 16 GB. Cannot resize.\n");
|
|
printf("Multi-slot via BAR1 resizing is NOT possible.\n");
|
|
printf("Fallback: NIC relocation to free 0x400000000 slot.\n");
|
|
} else if (size < 0x400000000ULL) {
|
|
printf("BAR1 minimum size: %llu GB — SMALLER THAN 16 GB!\n",
|
|
(unsigned long long)(size >> 30));
|
|
printf("Multi-slot via BAR1 resizing IS possible!\n");
|
|
printf("Could fit %llu dies in the 48 GB MMIO window.\n",
|
|
48ULL / (size >> 30));
|
|
} else {
|
|
printf("BAR1 size %llu GB — larger than expected\n",
|
|
(unsigned long long)(size >> 30));
|
|
}
|
|
|
|
/* Restore original BAR1 values */
|
|
write_pci_config(bdf, bar1_lo_off, orig_lo);
|
|
write_pci_config(bdf, bar1_hi_off, orig_hi);
|
|
|
|
/* Re-enable decode */
|
|
write_pci_config(bdf, 0x04, cmd);
|
|
printf("\nBAR1 restored, decode re-enabled.\n");
|
|
|
|
release:
|
|
/* Release die */
|
|
{
|
|
struct bs_release_req rr = { .die_id = die_id, .flags = 0 };
|
|
ioctl(bs_fd, BAR_SWAP_RELEASE, &rr);
|
|
printf("Die %d released.\n", die_id);
|
|
}
|
|
close(bs_fd);
|
|
return 0;
|
|
}
|