Skip to content

yaksa: use unsigned GPU device counts #7915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jeffhammond wants to merge 1 commit into
base: main
Choose a base branch
from
Open

yaksa: use unsigned GPU device counts #7915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions src/mpi/datatype/typerep/yaksa/src/backend/cuda/hooks/yaksuri_cuda_init_hooks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ static int finalize_hook(void)
int rc = YAKSA_SUCCESS;
cudaError_t cerr;

for (int i = 0; i < yaksuri_cudai_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_cudai_global.ndevices; i++) {
if (yaksuri_cudai_global.streams[i].created) {
cerr = cudaSetDevice(i);
cerr = cudaSetDevice((int) i);
YAKSURI_CUDAI_CUDA_ERR_CHKANDJUMP(cerr, rc, fn_fail);

cerr = cudaStreamDestroy(yaksuri_cudai_global.streams[i].stream);
Expand All @@ -93,7 +93,7 @@ static int finalize_hook(void)
goto fn_exit;
}

static int get_num_devices(int *ndevices)
static int get_num_devices(unsigned *ndevices)
{
*ndevices = yaksuri_cudai_global.ndevices;

Expand Down Expand Up @@ -169,21 +169,24 @@ int yaksuri_cuda_init_hook(yaksur_gpudriver_hooks_s ** hooks)
int rc = YAKSA_SUCCESS;
cudaError_t cerr;

cerr = cudaGetDeviceCount(&yaksuri_cudai_global.ndevices);
int ndevices;
cerr = cudaGetDeviceCount(&ndevices);
if (cerr == cudaErrorNoDevice || cerr == cudaErrorInsufficientDriver) {
/* both codes can indicate no devices available on the system */
goto fn_exit;
}
YAKSURI_CUDAI_CUDA_ERR_CHKANDJUMP(cerr, rc, fn_fail);
YAKSU_ERR_CHKANDJUMP(ndevices < 0, rc, YAKSA_ERR__INTERNAL, fn_fail);
yaksuri_cudai_global.ndevices = (unsigned) ndevices;

if (getenv("CUDA_VISIBLE_DEVICES") == NULL) {
/* user did not do any filtering for us; if any of the devices
* is in exclusive mode, disable GPU support to avoid
* incorrect device sharing */
bool excl = false;
for (int i = 0; i < yaksuri_cudai_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_cudai_global.ndevices; i++) {
int mode;
cerr = cudaDeviceGetAttribute(&mode, cudaDevAttrComputeMode, i);
cerr = cudaDeviceGetAttribute(&mode, cudaDevAttrComputeMode, (int) i);
YAKSURI_CUDAI_CUDA_ERR_CHKANDJUMP(cerr, rc, fn_fail);

if (mode != cudaComputeModeDefault) {
Expand All @@ -207,20 +210,20 @@ int yaksuri_cuda_init_hook(yaksur_gpudriver_hooks_s ** hooks)
yaksuri_cudai_global.streams = calloc(yaksuri_cudai_global.ndevices, sizeof(cudai_stream));

yaksuri_cudai_global.p2p = (int **) malloc(yaksuri_cudai_global.ndevices * sizeof(int *));
for (int i = 0; i < yaksuri_cudai_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_cudai_global.ndevices; i++) {
if (yaksuri_global.has_wait_kernel) {
/* The stream creation will deadlock with wait kernel. Create them now. */
yaksuri_cudai_get_stream(i);
yaksuri_cudai_get_stream((int) i);
}
yaksuri_cudai_global.p2p[i] = (int *) malloc(yaksuri_cudai_global.ndevices * sizeof(int));
/* mark as unchecked with -1. We will check access and cache the value
* in check_p2p_comm */
for (int j = 0; j < yaksuri_cudai_global.ndevices; j++) {
for (unsigned j = 0; j < yaksuri_cudai_global.ndevices; j++) {
yaksuri_cudai_global.p2p[i][j] = -1;
}
}
/* mark self entries */
for (int i = 0; i < yaksuri_cudai_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_cudai_global.ndevices; i++) {
yaksuri_cudai_global.p2p[i][i] = 1;
}

Expand Down
2 changes: 1 addition & 1 deletion src/mpi/datatype/typerep/yaksa/src/backend/cuda/include/yaksuri_cudai_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ typedef struct cudai_stream_s {
} cudai_stream;

typedef struct {
int ndevices;
unsigned ndevices;
cudai_stream *streams; /* array of lazily created streams, one for each device */
int **p2p; /* p2p[sdev][ddev] */
} yaksuri_cudai_global_s;
Expand Down
27 changes: 15 additions & 12 deletions src/mpi/datatype/typerep/yaksa/src/backend/hip/hooks/yaksuri_hip_init_hooks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ static int finalize_hook(void)
int rc = YAKSA_SUCCESS;
hipError_t cerr;

for (int i = 0; i < yaksuri_hipi_global.ndevices; i++) {
cerr = hipSetDevice(i);
for (unsigned i = 0; i < yaksuri_hipi_global.ndevices; i++) {
cerr = hipSetDevice((int) i);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

cerr = hipStreamDestroy(yaksuri_hipi_global.stream[i]);
Expand All @@ -82,7 +82,7 @@ static int finalize_hook(void)
goto fn_exit;
}

static int get_num_devices(int *ndevices)
static int get_num_devices(unsigned *ndevices)
{
*ndevices = yaksuri_hipi_global.ndevices;

Expand All @@ -107,21 +107,24 @@ int yaksuri_hip_init_hook(yaksur_gpudriver_hooks_s ** hooks)
int rc = YAKSA_SUCCESS;
hipError_t cerr;

cerr = hipGetDeviceCount(&yaksuri_hipi_global.ndevices);
int ndevices;
cerr = hipGetDeviceCount(&ndevices);
if (cerr == hipErrorNoDevice) {
goto fn_exit;
}
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);
YAKSU_ERR_CHKANDJUMP(ndevices < 0, rc, YAKSA_ERR__INTERNAL, fn_fail);
yaksuri_hipi_global.ndevices = (unsigned) ndevices;

if (getenv("HIP_VISIBLE_DEVICES") == NULL) {
/* user did not do any filtering for us; if any of the devices
* is in exclusive mode, disable GPU support to avoid
* incorrect device sharing */
bool excl = false;
for (int i = 0; i < yaksuri_hipi_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_hipi_global.ndevices; i++) {
struct hipDeviceProp_t prop;

cerr = hipGetDeviceProperties(&prop, i);
cerr = hipGetDeviceProperties(&prop, (int) i);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

if (prop.computeMode != hipComputeModeDefault) {
Expand All @@ -146,31 +149,31 @@ int yaksuri_hip_init_hook(yaksur_gpudriver_hooks_s ** hooks)
malloc(yaksuri_hipi_global.ndevices * sizeof(hipStream_t));

yaksuri_hipi_global.p2p = (bool **) malloc(yaksuri_hipi_global.ndevices * sizeof(bool *));
for (int i = 0; i < yaksuri_hipi_global.ndevices; i++) {
for (unsigned i = 0; i < yaksuri_hipi_global.ndevices; i++) {
yaksuri_hipi_global.p2p[i] = (bool *) malloc(yaksuri_hipi_global.ndevices * sizeof(bool));
}

int cur_device;
cerr = hipGetDevice(&cur_device);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

for (int i = 0; i < yaksuri_hipi_global.ndevices; i++) {
cerr = hipSetDevice(i);
for (unsigned i = 0; i < yaksuri_hipi_global.ndevices; i++) {
cerr = hipSetDevice((int) i);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

cerr = hipStreamCreateWithFlags(&yaksuri_hipi_global.stream[i], hipStreamNonBlocking);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

for (int j = 0; j < yaksuri_hipi_global.ndevices; j++) {
for (unsigned j = 0; j < yaksuri_hipi_global.ndevices; j++) {
if (i == j) {
yaksuri_hipi_global.p2p[i][j] = 1;
} else {
int val;
cerr = hipDeviceCanAccessPeer(&val, i, j);
cerr = hipDeviceCanAccessPeer(&val, (int) i, (int) j);
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);

if (val) {
cerr = hipDeviceEnablePeerAccess(j, 0);
cerr = hipDeviceEnablePeerAccess((int) j, 0);
if (cerr != hipErrorPeerAccessAlreadyEnabled) {
YAKSURI_HIPI_HIP_ERR_CHKANDJUMP(cerr, rc, fn_fail);
}
Expand Down
2 changes: 1 addition & 1 deletion src/mpi/datatype/typerep/yaksa/src/backend/hip/include/yaksuri_hipi_base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
} while (0)

typedef struct {
int ndevices;
unsigned ndevices;
hipStream_t *stream;
bool **p2p;
} yaksuri_hipi_global_s;
Expand Down
12 changes: 6 additions & 6 deletions src/mpi/datatype/typerep/yaksa/src/backend/src/yaksur_hooks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ static void *malloc_fn(uintptr_t size, void *state)

if (state == &yaksuri_global.gpudriver[id].host) {
return yaksuri_global.gpudriver[id].hooks->host_malloc(size);
} else {
} else if (yaksuri_global.gpudriver[id].ndevices > 0) {
uintptr_t start = (uintptr_t) yaksuri_global.gpudriver[id].device;
uintptr_t end =
(uintptr_t) & yaksuri_global.gpudriver[id].device[yaksuri_global.
Expand All @@ -49,7 +49,7 @@ static void free_fn(void *buf, void *state)
if (state == &yaksuri_global.gpudriver[id].host) {
yaksuri_global.gpudriver[id].hooks->host_free(buf);
return;
} else {
} else if (yaksuri_global.gpudriver[id].ndevices > 0) {
uintptr_t start = (uintptr_t) yaksuri_global.gpudriver[id].device;
uintptr_t end =
(uintptr_t) & yaksuri_global.gpudriver[id].device[yaksuri_global.
Expand Down Expand Up @@ -109,13 +109,13 @@ int yaksur_init_hook(yaksi_info_s * info)
&yaksuri_global.gpudriver[id].host);
YAKSU_ERR_CHECK(rc, fn_fail);

int ndevices;
unsigned ndevices;
rc = yaksuri_global.gpudriver[id].hooks->get_num_devices(&ndevices);
YAKSU_ERR_CHECK(rc, fn_fail);

yaksuri_global.gpudriver[id].device = (yaksu_buffer_pool_s *)
malloc(ndevices * sizeof(yaksu_buffer_pool_s));
for (int i = 0; i < ndevices; i++) {
for (unsigned i = 0; i < ndevices; i++) {
rc = yaksu_buffer_pool_alloc(YAKSURI_TMPBUF_EL_SIZE, 1, YAKSURI_TMPBUF_NUM_EL,
malloc_fn, free_fn,
&yaksuri_global.gpudriver[id].device[i],
Expand Down Expand Up @@ -153,8 +153,8 @@ int yaksur_finalize_hook(void)
rc = yaksu_buffer_pool_free(yaksuri_global.gpudriver[id].host);
YAKSU_ERR_CHECK(rc, fn_fail);

int ndevices = yaksuri_global.gpudriver[id].ndevices;
for (int i = 0; i < ndevices; i++) {
unsigned ndevices = yaksuri_global.gpudriver[id].ndevices;
for (unsigned i = 0; i < ndevices; i++) {
rc = yaksu_buffer_pool_free(yaksuri_global.gpudriver[id].device[i]);
YAKSU_ERR_CHECK(rc, fn_fail);
}
Expand Down
2 changes: 1 addition & 1 deletion src/mpi/datatype/typerep/yaksa/src/backend/src/yaksur_pre.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ typedef void (*yaksur_hostfn_t) (void *userData);

typedef struct yaksur_gpudriver_hooks_s {
/* miscellaneous */
int (*get_num_devices) (int *ndevices);
int (*get_num_devices) (unsigned *ndevices);
/* *INDENT-OFF* */
bool (*check_p2p_comm) (int sdev, int ddev);
/* *INDENT-ON* */
Expand Down
2 changes: 1 addition & 1 deletion src/mpi/datatype/typerep/yaksa/src/backend/src/yaksuri.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ typedef struct {
yaksu_buffer_pool_s host;
yaksu_buffer_pool_s *device;
yaksur_gpudriver_hooks_s *hooks;
int ndevices;
unsigned ndevices;
} gpudriver[YAKSURI_GPUDRIVER_ID__LAST];
} yaksuri_global_s;
extern yaksuri_global_s yaksuri_global;
Expand Down
2 changes: 1 addition & 1 deletion src/mpi/datatype/typerep/yaksa/src/backend/ze/hooks/yaksuri_ze_init_hooks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ static int finalize_hook(void)
goto fn_exit;
}

static int get_num_devices(int *ndevices)
static int get_num_devices(unsigned *ndevices)
{
*ndevices = yaksuri_zei_global.ndevices;

Expand Down