Skip to content

[runner] Drop buildLDLibraryPathEnv() #3593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
un-def wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[runner] Drop buildLDLibraryPathEnv() #3593

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 1 addition & 45 deletions runner/internal/executor/executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,15 +444,6 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error
"DSTACK_MPI_HOSTFILE": mpiHostfilePath,
}

// Call buildLDLibraryPathEnv and update jobEnvs if no error occurs
newLDPath, err := buildLDLibraryPathEnv(ctx)
if err != nil {
log.Info(ctx, "Continuing without updating LD_LIBRARY_PATH")
} else {
jobEnvs["LD_LIBRARY_PATH"] = newLDPath
log.Info(ctx, "New LD_LIBRARY_PATH set", "LD_LIBRARY_PATH", newLDPath)
}

cmd := exec.CommandContext(ctx, ex.jobSpec.Commands[0], ex.jobSpec.Commands[1:]...)
cmd.Cancel = func() error {
// returns error on Windows
Expand Down Expand Up @@ -504,8 +495,7 @@ func (ex *RunExecutor) execJob(ctx context.Context, jobLogFile io.Writer) error
log.Warning(ctx, "failed to include dstack_profile", "path", profilePath, "err", err)
}

err = writeMpiHostfile(ctx, ex.clusterInfo.JobIPs, gpusPerNodeNum, mpiHostfilePath)
if err != nil {
if err := writeMpiHostfile(ctx, ex.clusterInfo.JobIPs, gpusPerNodeNum, mpiHostfilePath); err != nil {
return fmt.Errorf("write MPI hostfile: %w", err)
}

Expand Down Expand Up @@ -621,40 +611,6 @@ func isPtyError(err error) bool {
return errors.As(err, &e) && errors.Is(e.Err, syscall.EIO)
}

func buildLDLibraryPathEnv(ctx context.Context) (string, error) {
// Execute shell command to get Python prefix
cmd := exec.CommandContext(ctx, "bash", "-i", "-c", "python3-config --prefix")
output, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("error executing command: %w", err)
}

// Extract and trim the prefix path
prefixPath := strings.TrimSpace(string(output))

// Check if the prefix path exists
if _, err := os.Stat(prefixPath); os.IsNotExist(err) {
return "", fmt.Errorf("python prefix path does not exist: %s", prefixPath)
}

// Construct the path to Python's shared libraries
sharedLibPath := fmt.Sprintf("%s/lib", prefixPath)

// Get current LD_LIBRARY_PATH
currentLDPath := os.Getenv("LD_LIBRARY_PATH")

// Append Python's shared library path if not already present
if !strings.Contains(currentLDPath, sharedLibPath) {
if currentLDPath == "" {
currentLDPath = sharedLibPath
} else {
currentLDPath = fmt.Sprintf("%s:%s", currentLDPath, sharedLibPath)
}
}

return currentLDPath, nil
}

// A simplified copypasta of creack/pty Start->StartWithSize->StartWithAttrs
// with two additions:
// * controlling terminal is properly set (cmd.Extrafiles, Cmd.SysProcAttr.Ctty)
Expand Down