Skip to content

Add rotation via composition for devices that don't support rotation at scanout #2228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matte-schwartz wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add rotation via composition for devices that don't support rotation at scanout #2228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c222e5f
Add composited output rotation for displays that can't rotate at scanout
matte-schwartz Jun 21, 2026
0a0122c
steamcompmgr: classify the interactive overlay by output width, not a…
matte-schwartz Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 46 additions & 2 deletions src/Backends/DRMBackend.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ namespace gamescope

static std::optional<CDRMAtomicProperty> Instantiate( const char *pszName, CDRMAtomicObject *pObject, const DRMObjectRawProperties& rawProperties );

uint32_t GetPropertyId() const { return m_uPropertyId; }
uint64_t GetPendingValue() const { return m_ulPendingValue; }
uint64_t GetCurrentValue() const { return m_ulCurrentValue; }
uint64_t GetInitialValue() const { return m_ulInitialValue; }
Expand Down Expand Up @@ -1934,7 +1935,7 @@ LiftoffStateCacheEntry FrameInfoToLiftoffStateCacheEntry( struct drm_t *drm, con
uint64_t crtcW = srcWidth / frameInfo->layers[ i ].scale.x;
uint64_t crtcH = srcHeight / frameInfo->layers[ i ].scale.y;

if (g_bRotated)
if (g_bRotated && g_uOutputRotation == 0)
{
int64_t imageH = frameInfo->layers[ i ].tex->contentHeight() / frameInfo->layers[ i ].scale.y;

Expand Down Expand Up @@ -2668,7 +2669,7 @@ drm_prepare_liftoff( struct drm_t *drm, const struct FrameInfo_t *frameInfo, boo
liftoff_layer_set_property( drm->lo_layers[ i ], "SRC_H", entry.layerState[i].srcH );

uint64_t ulOrientation = DRM_MODE_ROTATE_0;
switch ( drm->pConnector->GetCurrentOrientation() )
switch ( g_uOutputRotation != 0 ? GAMESCOPE_PANEL_ORIENTATION_0 : drm->pConnector->GetCurrentOrientation() )
{
default:
case GAMESCOPE_PANEL_ORIENTATION_0:
Expand Down Expand Up @@ -3278,6 +3279,29 @@ static void drm_unset_mode( struct drm_t *drm )
g_nDynamicRefreshHz = 0;

g_bRotated = false;
g_uOutputRotation = 0;
}

// Bitmask of DRM_MODE_ROTATE_* the plane can do at scanout (just ROTATE_0 if it can't rotate).
static uint64_t drm_plane_supported_rotations( struct drm_t *drm, gamescope::CDRMPlane *pPlane )
{
if ( !pPlane->GetProperties().rotation )
return DRM_MODE_ROTATE_0;

drmModePropertyRes *pProp = drmModeGetProperty( drm->fd, pPlane->GetProperties().rotation->GetPropertyId() );
if ( !pProp )
return DRM_MODE_ROTATE_0;
defer( drmModeFreeProperty( pProp ) );

if ( !( pProp->flags & DRM_MODE_PROP_BITMASK ) )
return DRM_MODE_ROTATE_0;

uint64_t ulSupported = 0;
for ( int i = 0; i < pProp->count_enums; i++ )
if ( pProp->enums[i].value < 64 )
ulSupported |= 1ull << pProp->enums[i].value;

return ulSupported;
}

bool drm_set_mode( struct drm_t *drm, const drmModeModeInfo *mode )
Expand Down Expand Up @@ -3312,6 +3336,25 @@ bool drm_set_mode( struct drm_t *drm, const drmModeModeInfo *mode )
break;
}

// Rotate in the compositor when the scanout plane can't do the panel's
// orientation. 90/270 transpose the output (g_bRotated); 180 flips in place.
g_uOutputRotation = 0;
uint32_t uStep = 0;
uint64_t uNeeded = 0;
switch ( drm->pConnector->GetCurrentOrientation() )
{
case GAMESCOPE_PANEL_ORIENTATION_90: uStep = 1u; uNeeded = DRM_MODE_ROTATE_90; break;
case GAMESCOPE_PANEL_ORIENTATION_180: uStep = 2u; uNeeded = DRM_MODE_ROTATE_180; break;
case GAMESCOPE_PANEL_ORIENTATION_270: uStep = 3u; uNeeded = DRM_MODE_ROTATE_270; break;
default: break;
}
if ( uStep )
{
const bool bScanoutCanRotate = drm->pPrimaryPlane && ( drm_plane_supported_rotations( drm, drm->pPrimaryPlane ) & uNeeded );
if ( g_bForceCompositionRotation || !bScanoutCanRotate )
g_uOutputRotation = uStep;
}

return true;
}

Expand Down Expand Up @@ -3566,6 +3609,7 @@ namespace gamescope
}

bNeedsFullComposite |= !!(g_uCompositeDebug & CompositeDebugFlag::Heatmap);
bNeedsFullComposite |= g_uOutputRotation != 0; // can't rotate planes at scanout

bool bDoComposite = true;
if ( !bNeedsFullComposite && !bWantsPartialComposite )
Expand Down
7 changes: 7 additions & 0 deletions src/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ const struct option *gamescope_options = (struct option[]){
{ "composite-debug", no_argument, nullptr, 0 },
{ "disable-xres", no_argument, nullptr, 'x' },
{ "fade-out-duration", required_argument, nullptr, 0 },
{ "force-composition-rotation", no_argument, nullptr, 0 },
{ "force-orientation", required_argument, nullptr, 0 },
{ "force-windows-fullscreen", no_argument, nullptr, 0 },

Expand Down Expand Up @@ -201,6 +202,7 @@ const char usage[] =
" -e, --steam enable Steam integration\n"
" --xwayland-count create N xwayland servers\n"
" --prefer-vk-device prefer Vulkan device for compositing (ex: 1002:7300)\n"
" --force-composition-rotation always rotate the output in the compositor instead of at scanout (autodetected otherwise)\n"
" --force-orientation rotate the internal display (left, right, normal, upsidedown)\n"
" --force-windows-fullscreen force windows inside of gamescope to be the size of the nested display (fullscreen)\n"
" --cursor-scale-height if specified, sets a base output height to linearly scale the cursor against.\n"
Expand Down Expand Up @@ -366,6 +368,9 @@ static gamescope::GamescopeModeGeneration parse_gamescope_mode_generation( const
}
}

bool g_bForceCompositionRotation = false;
uint32_t g_uOutputRotation = 0;

GamescopePanelOrientation g_DesiredInternalOrientation = GAMESCOPE_PANEL_ORIENTATION_AUTO;
static GamescopePanelOrientation force_orientation(const char *str)
{
Expand Down Expand Up @@ -798,6 +803,8 @@ int main(int argc, char **argv)
gamescope::cv_touch_click_mode = (gamescope::TouchClickMode) parse_integer( optarg, opt_name );
} else if (strcmp(opt_name, "generate-drm-mode") == 0) {
g_eGamescopeModeGeneration = parse_gamescope_mode_generation( optarg );
} else if (strcmp(opt_name, "force-composition-rotation") == 0) {
g_bForceCompositionRotation = true;
} else if (strcmp(opt_name, "force-orientation") == 0) {
g_DesiredInternalOrientation = force_orientation( optarg );
} else if (strcmp(opt_name, "sharpness") == 0 ||
Expand Down
3 changes: 3 additions & 0 deletions src/main.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ extern int g_nOutputRefresh; // mHz
extern bool g_bOutputHDREnabled;
extern bool g_bForceInternal;

extern bool g_bForceCompositionRotation;
extern uint32_t g_uOutputRotation;

extern bool g_bFullscreen;

extern bool g_bGrabbed;
Expand Down
40 changes: 28 additions & 12 deletions src/rendervulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3300,24 +3300,30 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput )

uint32_t uDRMFormat = pOutput->uOutputFormat;

// Output images are physical-oriented; the panel scans out unrotated.
uint32_t uOutputWidth = g_nOutputWidth;
uint32_t uOutputHeight = g_nOutputHeight;
if ( g_uOutputRotation & 1u )
std::swap( uOutputWidth, uOutputHeight );

pOutput->outputImages[0] = new CVulkanTexture();
bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
bool bSuccess = pOutput->outputImages[0]->BInit( uOutputWidth, uOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImages[1] = new CVulkanTexture();
bSuccess = pOutput->outputImages[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
bSuccess = pOutput->outputImages[1]->BInit( uOutputWidth, uOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImages[2] = new CVulkanTexture();
bSuccess = pOutput->outputImages[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags );
bSuccess = pOutput->outputImages[2]->BInit( uOutputWidth, uOutputHeight, 1u, uDRMFormat, outputImageflags );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
Expand All @@ -3332,23 +3338,23 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput )
uint32_t uPartialDRMFormat = pOutput->uOutputFormatOverlay;

pOutput->outputImagesPartialOverlay[0] = new CVulkanTexture();
bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() );
bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( uOutputWidth, uOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImagesPartialOverlay[1] = new CVulkanTexture();
bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() );
bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( uOutputWidth, uOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
return false;
}

pOutput->outputImagesPartialOverlay[2] = new CVulkanTexture();
bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() );
bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( uOutputWidth, uOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() );
if ( bSuccess != true )
{
vk_log.errorf( "failed to allocate buffer for KMS" );
Expand Down Expand Up @@ -3703,10 +3709,13 @@ struct BlitPushData_t
float u_itmSdrNits; // unset
float u_itmTargetNits; // unset

explicit BlitPushData_t(const struct FrameInfo_t *frameInfo)
uint32_t u_rotation;

explicit BlitPushData_t(const struct FrameInfo_t *frameInfo, uint32_t rotation = 0)
{
u_shaderFilter = 0;
u_alphaMode = 0;
u_rotation = rotation;

for (int i = 0; i < frameInfo->layerCount; i++) {
const FrameInfo_t::Layer_t *layer = &frameInfo->layers[i];
Expand Down Expand Up @@ -3751,6 +3760,7 @@ struct BlitPushData_t
opacity[0] = 1.0f;
u_shaderFilter = (uint32_t)GamescopeUpscaleFilter::LINEAR;
u_alphaMode = 0;
u_rotation = 0;
ctm[0] = glm::mat3x4
{
1, 0, 0, 0,
Expand Down Expand Up @@ -3837,10 +3847,13 @@ struct RcasPushData_t
float u_itmSdrNits; // unset
float u_itmTargetNits; // unset

RcasPushData_t(const struct FrameInfo_t *frameInfo, float sharpness)
uint32_t u_rotation;

RcasPushData_t(const struct FrameInfo_t *frameInfo, float sharpness, uint32_t rotation = 0)
{
uvec4_t tmp;
FsrRcasCon(&tmp.x, sharpness);
u_rotation = rotation;
u_layer0Offset.x = uint32_t(int32_t(frameInfo->layers[0].offset.x));
u_layer0Offset.y = uint32_t(int32_t(frameInfo->layers[0].offset.y));
u_borderMask = frameInfo->borderMask() >> 1u;
Expand Down Expand Up @@ -4031,6 +4044,9 @@ std::optional<uint64_t> vulkan_composite( struct FrameInfo_t *frameInfo, gamesco
else
compositeImage = partial ? g_output.outputImagesPartialOverlay[ g_output.nOutImage ] : g_output.outputImages[ g_output.nOutImage ];

// Overrides (screenshots, upscale cache) are logical-sized, so never rotated.
const uint32_t uOutputRotation = pOutputOverride ? 0u : g_uOutputRotation;

auto cmdBuffer = pInCommandBuffer ? std::move( pInCommandBuffer ) : g_device.commandBuffer();

for (uint32_t i = 0; i < EOTF_Count; i++)
Expand Down Expand Up @@ -4065,7 +4081,7 @@ std::optional<uint64_t> vulkan_composite( struct FrameInfo_t *frameInfo, gamesco
cmdBuffer->setSamplerUnnormalized(0, false);
cmdBuffer->setSamplerNearest(0, false);
cmdBuffer->bindTarget(compositeImage);
cmdBuffer->uploadConstants<RcasPushData_t>(frameInfo, g_upscaleFilterSharpness / 10.0f);
cmdBuffer->uploadConstants<RcasPushData_t>(frameInfo, g_upscaleFilterSharpness / 10.0f, uOutputRotation);

cmdBuffer->dispatch(div_roundup(currentOutputWidth, pixelsPerGroup), div_roundup(currentOutputHeight, pixelsPerGroup));
}
Expand Down Expand Up @@ -4108,7 +4124,7 @@ std::optional<uint64_t> vulkan_composite( struct FrameInfo_t *frameInfo, gamesco
cmdBuffer->bindPipeline( g_device.pipeline(SHADER_TYPE_BLIT, nisFrameInfo.layerCount, nisFrameInfo.ycbcrMask(), 0u, nisFrameInfo.colorspaceMask(), outputTF ));
bind_all_layers(cmdBuffer.get(), &nisFrameInfo);
cmdBuffer->bindTarget(compositeImage);
cmdBuffer->uploadConstants<BlitPushData_t>(&nisFrameInfo);
cmdBuffer->uploadConstants<BlitPushData_t>(&nisFrameInfo, uOutputRotation);

int pixelsPerGroup = 8;

Expand All @@ -4134,7 +4150,7 @@ std::optional<uint64_t> vulkan_composite( struct FrameInfo_t *frameInfo, gamesco
cmdBuffer->setSamplerUnnormalized(i, true);
cmdBuffer->setSamplerNearest(i, false);
}
cmdBuffer->uploadConstants<BlitPushData_t>(frameInfo);
cmdBuffer->uploadConstants<BlitPushData_t>(frameInfo, uOutputRotation);

int pixelsPerGroup = 8;

Expand All @@ -4158,7 +4174,7 @@ std::optional<uint64_t> vulkan_composite( struct FrameInfo_t *frameInfo, gamesco
cmdBuffer->bindPipeline( g_device.pipeline(SHADER_TYPE_BLIT, frameInfo->layerCount, frameInfo->ycbcrMask(), 0u, frameInfo->colorspaceMask(), outputTF ));
bind_all_layers(cmdBuffer.get(), frameInfo);
cmdBuffer->bindTarget(compositeImage);
cmdBuffer->uploadConstants<BlitPushData_t>(frameInfo);
cmdBuffer->uploadConstants<BlitPushData_t>(frameInfo, uOutputRotation);

const int pixelsPerGroup = 8;

Expand Down
2 changes: 2 additions & 0 deletions src/shaders/blit_push_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,7 @@ uniform layers_t {
float u_nitsToLinear; // hdr -> sdr
float u_itmSdrNits;
float u_itmTargetNits;

uint u_rotation;
};

21 changes: 19 additions & 2 deletions src/shaders/composite.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,23 @@
#include "shaderfilter.h"
#include "alphamode.h"

// Rotate only the final store coordinate (CCW 90-degree steps: 1=90, 2=180,
// 3=270); the scene stays logical so sampling and blending are unchanged.
uvec2 outputLogicalSize(uint rotation) {
uvec2 size = imageSize(dst);
return (rotation & 1u) != 0u ? uvec2(size.y, size.x) : size;
}

ivec2 rotateOutputCoord(uvec2 coord, uint rotation) {
uvec2 size = imageSize(dst);
switch (rotation) {
case 1u: return ivec2(coord.y, size.y - 1u - coord.x);
case 2u: return ivec2(size.x - 1u - coord.x, size.y - 1u - coord.y);
case 3u: return ivec2(size.x - 1u - coord.y, coord.x);
default: return ivec2(coord);
}
}

vec4 sampleRegular(sampler2D tex, vec2 coord, uint colorspace) {
vec4 color = textureLod(tex, coord, 0);
color.rgb = colorspace_plane_degamma_tf(color.rgb, colorspace);
Expand Down Expand Up @@ -50,7 +67,7 @@ uint pseudo_random(uint seed) {
return seed * 1664525u + 1013904223u;
}

void compositing_debug(uvec2 coord) {
void compositing_debug(uvec2 coord, uint rotation) {
uvec2 pos = coord;
pos.x -= (u_frameId & 2) != 0 ? 128 : 0;
pos.y -= (u_frameId & 1) != 0 ? 128 : 0;
Expand All @@ -66,7 +83,7 @@ void compositing_debug(uvec2 coord) {
if (time.x + time.y + time.z + time.w < 2.0f)
value = vec4(0.0f, 0.0f, 0.0f, 1.0f);
}
imageStore(dst, ivec2(coord), value);
imageStore(dst, rotateOutputCoord(coord, rotation), value);
}
}

Expand Down
6 changes: 3 additions & 3 deletions src/shaders/cs_composite_blit.comp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ vec4 sampleLayer(uint layerIdx, vec2 uv) {

void main() {
uvec2 coord = uvec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y);
uvec2 outSize = imageSize(dst);
uvec2 outSize = outputLogicalSize(u_rotation);

if (coord.x >= outSize.x || coord.y >= outSize.y)
return;
Expand All @@ -42,9 +42,9 @@ void main() {
}

outputValue.rgb = encodeOutputColor(outputValue.rgb);
imageStore(dst, ivec2(coord), outputValue);
imageStore(dst, rotateOutputCoord(coord, u_rotation), outputValue);

// Indicator to quickly tell if we're in the compositing path or not.
if (checkDebugFlag(compositedebug_Markers))
compositing_debug(coord);
compositing_debug(coord, u_rotation);
}
6 changes: 3 additions & 3 deletions src/shaders/cs_composite_blur.comp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ vec4 sampleLayer(uint layerIdx, vec2 uv) {

void main() {
uvec2 coord = uvec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y);
uvec2 outSize = imageSize(dst);
uvec2 outSize = outputLogicalSize(u_rotation);

if (coord.x >= outSize.x || coord.y >= outSize.y)
return;
Expand All @@ -44,8 +44,8 @@ void main() {
}

outputValue = encodeOutputColor(outputValue);
imageStore(dst, ivec2(coord), vec4(outputValue, 0));
imageStore(dst, rotateOutputCoord(coord, u_rotation), vec4(outputValue, 0));

if (checkDebugFlag(compositedebug_Markers))
compositing_debug(coord);
compositing_debug(coord, u_rotation);
}
6 changes: 3 additions & 3 deletions src/shaders/cs_composite_blur_cond.comp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ vec4 sampleLayer(uint layerIdx, vec2 uv) {

void main() {
uvec2 coord = uvec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y);
uvec2 outSize = imageSize(dst);
uvec2 outSize = outputLogicalSize(u_rotation);

if (coord.x >= outSize.x || coord.y >= outSize.y)
return;
Expand Down Expand Up @@ -61,8 +61,8 @@ void main() {
}

outputValue = encodeOutputColor(outputValue);
imageStore(dst, ivec2(coord), vec4(outputValue, 0));
imageStore(dst, rotateOutputCoord(coord, u_rotation), vec4(outputValue, 0));

if (checkDebugFlag(compositedebug_Markers))
compositing_debug(coord);
compositing_debug(coord, u_rotation);
}
Loading