Add resizable virtual display feature

Introduce `--flex-display` to continuously resize the virtual display
to match the window.

app/data/bash-completion/scrcpy

@@ -108,7 +108,8 @@ _scrcpy() {
         --window-x=
         --window-y=
         --window-width=
-        --window-height="
+        --window-height=
+        -x --flex-display"
 
     _init_completion -s || return
 

app/data/zsh-completion/_scrcpy

@@ -113,6 +113,7 @@ arguments=(
     '--window-y=[Set the initial window vertical position]'
     '--window-width=[Set the initial window width]'
     '--window-height=[Set the initial window height]'
+    {-x,--flex-display}'[Continuously resize the virtual display to match the window]'
 )
 
 _arguments -s $arguments

app/scrcpy.1

@@ -523,7 +523,7 @@ Possible values are "letterbox" and "disabled":
  - "letterbox": preserve the aspect ratio and fit the window as best as possible (black bars are added either at the top and bottom or at the sides if needed).
  - "disabled": render the display at the top-left corner, without scaling.
 
-Default is "letterbox".
+Default is "letterbox", unless --flex-display is set, in which case it is "disabled".
 
 .TP
 .B \-\-require\-audio
@@ -695,6 +695,10 @@ Set the initial window height.
 
 Default is 0 (automatic).
 
+.TP
+.B \-x, \-\-flex\-display
+Continuously resize the virtual display to match the window.
+
 .SH EXIT STATUS
 .B scrcpy
 will exit with code 0 on normal program termination. If an initial

app/src/cli.c

@@ -791,7 +791,8 @@ static const struct sc_option options[] = {
                 "and bottom or at the sides if needed).\n"
                 "\"disabled\": render the display at the top-left corner, "
                 "without scaling.\n"
-                "Default is \"letterbox\".",
+                "Default is \"letterbox\", unless --flex-display is set, in "
+                "which case it is \"disabled\".",
     },
     {
         .longopt_id = OPT_REQUIRE_AUDIO,
@@ -1016,6 +1017,11 @@ static const struct sc_option options[] = {
         .text = "Set the initial window height.\n"
                 "Default is 0 (automatic).",
     },
+    {
+        .shortopt = 'x',
+        .longopt = "flex-display",
+        .text = "Continuously resize the virtual display to match the window.",
+    },
 };
 
 static const struct sc_shortcut shortcuts[] = {
@@ -2792,6 +2798,9 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
                     return false;
                 }
                 break;
+            case 'x':
+                opts->flex_display = true;
+                break;
             default:
                 // getopt prints the error message on stderr
                 return false;
@@ -2971,6 +2980,11 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         }
     }
 
+    if (opts->render_fit == SC_RENDER_FIT_AUTO) {
+        opts->render_fit = opts->flex_display ? SC_RENDER_FIT_DISABLED
+                                              : SC_RENDER_FIT_LETTERBOX;
+    }
+
     if (otg) {
         if (!opts->control) {
             LOGE("--no-control is not allowed in OTG mode");
@@ -3090,6 +3104,28 @@ parse_args_with_getopt(struct scrcpy_cli_args *args, int argc, char *argv[],
         return false;
     }
 
+    if (opts->flex_display) {
+        if (opts->video_source != SC_VIDEO_SOURCE_DISPLAY
+                || !opts->new_display) {
+            LOGE("-x/--flex-display can only be applied to displays created "
+                 "with --new-display");
+            return false;
+        }
+
+        if (opts->max_size) {
+            LOGE("--max-size is not compatible with -x/--flex-display");
+            return false;
+        }
+
+        if (opts->crop) {
+            LOGE("--crop is not compatible with -x/--flex-display");
+            return false;
+        }
+
+        // Force free resizing
+        opts->window_aspect_ratio_lock = false;
+    }
+
     if (opts->display_ime_policy != SC_DISPLAY_IME_POLICY_UNDEFINED
             && opts->display_id == 0 && !opts->new_display) {
         LOGE("--display-ime-policy is only supported on a secondary display");

app/src/options.c

@@ -59,7 +59,7 @@ const struct scrcpy_options scrcpy_options_default = {
     .display_orientation = SC_ORIENTATION_0,
     .record_orientation = SC_ORIENTATION_0,
     .display_ime_policy = SC_DISPLAY_IME_POLICY_UNDEFINED,
-    .render_fit = SC_RENDER_FIT_LETTERBOX,
+    .render_fit = SC_RENDER_FIT_AUTO,
     .window_x = SC_WINDOW_POSITION_UNDEFINED,
     .window_y = SC_WINDOW_POSITION_UNDEFINED,
     .window_width = 0,
@@ -117,6 +117,7 @@ const struct scrcpy_options scrcpy_options_default = {
     .vd_destroy_content = true,
     .vd_system_decorations = true,
     .camera_torch = false,
+    .flex_display = false,
 };
 
 enum sc_orientation

app/src/options.h

@@ -221,6 +221,7 @@ enum sc_shortcut_mod {
 };
 
 enum sc_render_fit {
+    SC_RENDER_FIT_AUTO,
     SC_RENDER_FIT_LETTERBOX,
     SC_RENDER_FIT_DISABLED,
 };
@@ -336,6 +337,7 @@ struct scrcpy_options {
     bool vd_destroy_content;
     bool vd_system_decorations;
     bool camera_torch;
+    bool flex_display;
 };
 
 extern const struct scrcpy_options scrcpy_options_default;

app/src/scrcpy.c

@@ -465,6 +465,7 @@ scrcpy(struct scrcpy_options *options) {
         .camera_zoom = options->camera_zoom,
         .vd_destroy_content = options->vd_destroy_content,
         .vd_system_decorations = options->vd_system_decorations,
+        .flex_display = options->flex_display,
         .list = options->list,
     };
 
@@ -797,6 +798,7 @@ aoa_complete:
         struct sc_screen_params screen_params = {
             .video = options->video_playback,
             .camera = options->video_source == SC_VIDEO_SOURCE_CAMERA,
+            .flex_display = options->flex_display,
             .controller = controller,
             .fp = fp,
             .kp = kp,

app/src/screen.c

@@ -286,10 +286,27 @@ end:
 }
 
 static void
-sc_screen_on_resize(struct sc_screen *screen) {
+sc_screen_on_resize(struct sc_screen *screen, const SDL_WindowEvent *event) {
     // This event can be triggered before the window is shown
     if (screen->window_shown) {
         sc_screen_render(screen, true);
+
+        if (screen->flex_display) {
+            assert(!screen->camera);
+            assert(!(event->data1 & !0xFFFF));
+            assert(!(event->data2 & !0xFFFF));
+            uint16_t width = event->data1;
+            uint16_t height = event->data2;
+            if (sc_orientation_is_swap(screen->orientation)) {
+                uint16_t tmp = width;
+                width = height;
+                height = tmp;
+            }
+
+            LOGV("resize_display(%" PRIu16 ", %" PRIu16 ")", width, height);
+            sc_controller_resize_display(screen->controller, width,
+                                         height);
+        }
     }
 }
 
@@ -311,7 +328,7 @@ event_watcher(void *data, SDL_Event *event) {
     if (event->type == SDL_EVENT_WINDOW_PIXEL_SIZE_CHANGED) {
         // In practice, it seems to always be called from the same thread in
         // that specific case. Anyway, it's just a workaround.
-        sc_screen_on_resize(screen);
+        sc_screen_on_resize(screen, &event->window);
     }
 
     return true;
@@ -406,6 +423,8 @@ sc_screen_frame_sink_push_session(struct sc_frame_sink *sink,
 bool
 sc_screen_init(struct sc_screen *screen,
                const struct sc_screen_params *params) {
+    screen->controller = params->controller;
+
     screen->resize_pending = false;
     screen->window_shown = false;
     screen->paused = false;
@@ -418,6 +437,7 @@ sc_screen_init(struct sc_screen *screen,
     screen->camera = params->camera;
     screen->window_aspect_ratio_lock = params->window_aspect_ratio_lock;
     screen->render_fit = params->render_fit;
+    screen->flex_display = params->flex_display;
 
     screen->req.x = params->window_x;
     screen->req.y = params->window_y;
@@ -739,11 +759,13 @@ resize_for_content(struct sc_screen *screen, struct sc_size old_content_size,
     assert(screen->video);
 
     struct sc_size window_size = sc_sdl_get_window_size(screen->window);
-    struct sc_size target_size = {
-        .width = (uint32_t) window_size.width * new_content_size.width
-                / old_content_size.width,
-        .height = (uint32_t) window_size.height * new_content_size.height
-                / old_content_size.height,
+    struct sc_size target_size = new_content_size;
+    if (!screen->flex_display) {
+        // Scale proportionally
+        target_size.width = (uint32_t) window_size.width * target_size.width
+                          / old_content_size.width;
+        target_size.height = (uint32_t) window_size.height * target_size.height
+                           / old_content_size.height;
     };
     target_size = get_optimal_size(target_size, new_content_size, true);
     assert(is_windowed(screen));
@@ -999,7 +1021,7 @@ sc_screen_handle_event(struct sc_screen *screen, const SDL_Event *event) {
 // If defined, then the actions are already performed by the event watcher
 #ifndef CONTINUOUS_RESIZING_WORKAROUND
         case SDL_EVENT_WINDOW_PIXEL_SIZE_CHANGED:
-            sc_screen_on_resize(screen);
+            sc_screen_on_resize(screen, &event->window);
             return;
 #endif
         case SDL_EVENT_WINDOW_RESTORED:

app/src/screen.h

@@ -37,6 +37,9 @@ struct sc_screen {
     bool video;
     bool camera;
     bool window_aspect_ratio_lock;
+    bool flex_display;
+
+    struct sc_controller *controller;
 
     struct sc_texture tex;
     struct sc_input_manager im;
@@ -96,6 +99,7 @@ struct sc_screen {
 struct sc_screen_params {
     bool video;
     bool camera;
+    bool flex_display;
 
     struct sc_controller *controller;
     struct sc_file_pusher *fp;

app/src/server.c

@@ -413,6 +413,9 @@ execute_server(struct sc_server *server,
         VALIDATE_STRING(params->new_display);
         ADD_PARAM("new_display=%s", params->new_display);
     }
+    if (params->flex_display) {
+        ADD_PARAM("flex_display=true");
+    }
     if (params->display_ime_policy != SC_DISPLAY_IME_POLICY_UNDEFINED) {
         ADD_PARAM("display_ime_policy=%s",
             sc_server_get_display_ime_policy_name(params->display_ime_policy));

app/src/server.h

@@ -72,6 +72,7 @@ struct sc_server_params {
     bool camera_torch;
     bool vd_destroy_content;
     bool vd_system_decorations;
+    bool flex_display;
     uint8_t list;
 };
 

server/src/main/java/com/genymobile/scrcpy/Options.java

@@ -66,6 +66,7 @@ public class Options {
     private NewDisplay newDisplay;
     private boolean vdDestroyContent = true;
     private boolean vdSystemDecorations = true;
+    private boolean flexDisplay;
 
     private Orientation.Lock captureOrientationLock = Orientation.Lock.Unlocked;
     private Orientation captureOrientation = Orientation.Orient0;
@@ -258,6 +259,10 @@ public class Options {
         return vdSystemDecorations;
     }
 
+    public boolean getFlexDisplay() {
+        return flexDisplay;
+    }
+
     public boolean getList() {
         return listEncoders || listDisplays || listCameras || listCameraSizes || listApps;
     }
@@ -506,6 +511,9 @@ public class Options {
                 case "vd_system_decorations":
                     options.vdSystemDecorations = Boolean.parseBoolean(value);
                     break;
+                case "flex_display":
+                    options.flexDisplay = Boolean.parseBoolean(value);
+                    break;
                 case "capture_orientation":
                     Pair<Orientation.Lock, Orientation> pair = parseCaptureOrientation(value);
                     options.captureOrientationLock = pair.first;

server/src/main/java/com/genymobile/scrcpy/control/Controller.java

@@ -14,6 +14,7 @@ import com.genymobile.scrcpy.util.Ln;
 import com.genymobile.scrcpy.util.LogUtils;
 import com.genymobile.scrcpy.video.CameraCapture;
 import com.genymobile.scrcpy.video.CaptureControl;
+import com.genymobile.scrcpy.video.NewDisplayCapture;
 import com.genymobile.scrcpy.video.SurfaceCapture;
 import com.genymobile.scrcpy.video.VideoSource;
 import com.genymobile.scrcpy.video.VirtualDisplayListener;
@@ -370,6 +371,9 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
                 case ControlMessage.TYPE_START_APP:
                     startAppAsync(msg.getText());
                     return true;
+                case ControlMessage.TYPE_RESIZE_DISPLAY:
+                    resizeDisplay(msg.getWidth(), msg.getHeight());
+                    return true;
                 default:
                     // fall through
             }
@@ -819,4 +823,9 @@ public class Controller implements AsyncProcessor, VirtualDisplayListener {
             surfaceCapture.getCaptureControl().reset(CaptureControl.RESET_REASON_CLIENT_RESET);
         }
     }
+
+    private void resizeDisplay(int width, int height) {
+        NewDisplayCapture newDisplayCapture = (NewDisplayCapture) surfaceCapture;
+        newDisplayCapture.resizeDisplay(width, height);
+    }
 }

server/src/main/java/com/genymobile/scrcpy/model/Orientation.java

@@ -46,4 +46,9 @@ public enum Orientation {
     public int getRotation() {
         return ordinal() & 3;
     }
+
+    public boolean isSwap() {
+        // width and height are swapped on 90-degree rotations
+        return (ordinal() & 1) != 0;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/model/Size.java

@@ -93,6 +93,10 @@ public final class Size {
         return new Size(w, h);
     }
 
+    public boolean isAligned(int alignment) {
+        return width / alignment * alignment == width && height / alignment * alignment == height;
+    }
+
     private static int round(int value, int alignment) {
         return (value + (alignment / 2)) / alignment * alignment;
     }

server/src/main/java/com/genymobile/scrcpy/video/CaptureControl.java

@@ -7,6 +7,7 @@ public class CaptureControl {
     public static final int RESET_REASON_TERMINATE = 1;
     public static final int RESET_REASON_SIZE_CHANGED = 1 << 1;
     public static final int RESET_REASON_CLIENT_RESET = 1 << 2;
+    public static final int RESET_REASON_CLIENT_RESIZED = 1 << 2;
 
     private int reset = 0;
 

server/src/main/java/com/genymobile/scrcpy/video/NewDisplayCapture.java

@@ -58,12 +58,16 @@ public class NewDisplayCapture extends SurfaceCapture {
     private final float angle;
     private final boolean vdDestroyContent;
     private final boolean vdSystemDecorations;
+    private final boolean flexDisplay;
 
     private VirtualDisplay virtualDisplay;
     private Size videoSize;
     private Size displaySize; // the logical size of the display (including rotation)
     private Size physicalSize; // the physical size of the display (without rotation)
 
+    private Size pendingClientResized;
+    private Size latestSize;
+
     private int dpi;
 
     public NewDisplayCapture(VirtualDisplayListener vdListener, Options options) {
@@ -79,13 +83,30 @@ public class NewDisplayCapture extends SurfaceCapture {
         this.angle = options.getAngle();
         this.vdDestroyContent = options.getVDDestroyContent();
         this.vdSystemDecorations = options.getVDSystemDecorations();
+        this.flexDisplay = options.getFlexDisplay();
     }
 
     @Override
     protected void init() {
+        if (flexDisplay && crop != null) {
+            throw new IllegalArgumentException("Flex display does not support cropping");
+        }
+        if (getVideoConstraints().getMaxSize() != 0) {
+            // A maxSize request constrains the resulting size while preserving the aspect ratio, which is meaningless for a flex display
+            throw new IllegalArgumentException("Flex display does not support explicit maxSize constraint");
+        }
+
         displaySize = newDisplay.getSize();
         dpi = newDisplay.getDpi();
-        if (displaySize == null || dpi == 0) {
+        if (flexDisplay) {
+            // Hardcode default values if not defined
+            if (displaySize == null) {
+                displaySize = new Size(1280, 960);
+            }
+            if (dpi == 0) {
+                dpi = 160;
+            }
+        } else if (displaySize == null || dpi == 0) {
             DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(0);
             if (displayInfo != null) {
                 mainDisplaySize = displayInfo.getSize();
@@ -103,16 +124,20 @@ public class NewDisplayCapture extends SurfaceCapture {
 
     @Override
     public void prepare() {
+        VideoConstraints constraints = getVideoConstraints();
+
         int displayRotation;
         if (virtualDisplay == null) {
-            if (!newDisplay.hasExplicitSize()) {
+            if (displaySize == null) {
+                assert !flexDisplay;
                 displaySize = mainDisplaySize;
             }
 
             // Align the physical display size to avoid unnecessary mismatches with the output size
             displaySize = displaySize.align(getVideoConstraints().getAlignment());
 
-            if (!newDisplay.hasExplicitDpi()) {
+            if (dpi == 0) {
+                assert !flexDisplay;
                 dpi = scaleDpi(mainDisplaySize, mainDisplayDpi, displaySize);
             }
 
@@ -124,8 +149,27 @@ public class NewDisplayCapture extends SurfaceCapture {
             dpi = displayInfo.getDpi();
             displayRotation = displayInfo.getRotation();
 
-            // Align the physical display size to avoid unnecessary mismatches with the output size
-            displaySize = displayInfo.getSize().align(getVideoConstraints().getAlignment());
+            Size pendingClientResized = consumeClientResized();
+            if (pendingClientResized != null) {
+                assert pendingClientResized.isAligned(getVideoConstraints().getAlignment()) : "pendingClientResized ust be aligned";
+                displaySize = pendingClientResized;
+                if (captureOrientation.isSwap()) {
+                    displaySize = displaySize.rotate();
+                }
+                Size vdSize = displaySize;
+                if ((displayRotation % 2) != 0) {
+                    vdSize = vdSize.rotate();
+                }
+
+                displayMonitor.expectChange(new DisplayProperties(vdSize, displayRotation));
+                if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                    Ln.v(getClass().getSimpleName() + ": virtualDisplay.resize(" + vdSize.getWidth() + ", " + vdSize.getHeight() + ")");
+                }
+                virtualDisplay.resize(vdSize.getWidth(), vdSize.getHeight(), dpi);
+            } else {
+                // Align the physical display size to avoid unnecessary mismatches with the output size
+                displaySize = displayInfo.getSize().align(getVideoConstraints().getAlignment());
+            }
         }
 
         VideoFilter filter = new VideoFilter(displaySize);
@@ -139,7 +183,7 @@ public class NewDisplayCapture extends SurfaceCapture {
         filter.addAngle(angle);
 
         Size outputSize = filter.getOutputSize();
-        Size filteredSize = outputSize.constrain(getVideoConstraints());
+        Size filteredSize = outputSize.constrain(constraints);
         if (!filteredSize.equals(outputSize)) {
             filter.addResize(filteredSize);
         }
@@ -148,6 +192,7 @@ public class NewDisplayCapture extends SurfaceCapture {
 
         // DisplayInfo gives the oriented size (so videoSize includes the display rotation)
         videoSize = filter.getOutputSize();
+        setLatestSize(videoSize);
 
         // However, the virtual display video always remains in its original orientation, so it must be rotated manually.
         // This additional display rotation must not be included in the input events transform (the expected coordinates are already in the
@@ -257,4 +302,51 @@ public class NewDisplayCapture extends SurfaceCapture {
         int num = size.getMax();
         return initialDpi * num / den;
     }
+
+    public synchronized void resizeDisplay(int width, int height) {
+        if (!flexDisplay) {
+            throw new IllegalStateException("Cannot resize a non-flex display");
+        }
+
+        Size newSize = new Size(width, height).constrain(getVideoConstraints());
+        if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+            Ln.v(getClass().getSimpleName() + ": resizeDisplay(" + width + ", " + height + ")");
+            Ln.v(getClass().getSimpleName() + ": constrained size=" + newSize);
+        }
+        if (newSize.equals(pendingClientResized)) {
+            // Already requested
+            if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                Ln.v(getClass().getSimpleName() + ": new size already requested (" + newSize + ")");
+            }
+            return;
+        }
+
+        Size latestSize = getLatestSize(); // with synchro
+        if (newSize.equals(latestSize)) {
+            if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+                Ln.v(getClass().getSimpleName() + ": requested new size (" + newSize + ") is already the latest one");
+            }
+            return;
+        }
+
+        pendingClientResized = newSize;
+        if (Ln.isEnabled(Ln.Level.VERBOSE)) {
+            Ln.v(getClass().getSimpleName() + ": reset (" + newSize + ")");
+        }
+        getCaptureControl().reset(CaptureControl.RESET_REASON_CLIENT_RESIZED);
+    }
+
+    private synchronized Size consumeClientResized() {
+        Size size = pendingClientResized;
+        pendingClientResized = null;
+        return size;
+    }
+
+    private synchronized Size getLatestSize() {
+        return latestSize;
+    }
+
+    private synchronized void setLatestSize(Size latestSize) {
+        this.latestSize = latestSize;
+    }
 }

server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java

@@ -128,7 +128,11 @@ public class SurfaceEncoder implements AsyncProcessor {
                     } else {
                         if (!captureControl.isResetRequested()) {
                             // If a reset is requested during encode(), it will interrupt the encoding by an EOS
-                            streamer.writeSessionMeta(size.getWidth(), size.getHeight(), false);
+
+                            // The reset is due to a resize initiated by the client
+                            boolean clientResize = (resetReasons & CaptureControl.RESET_REASON_CLIENT_RESIZED) != 0
+                                    && (resetReasons & CaptureControl.RESET_REASON_SIZE_CHANGED) == 0;
+                            streamer.writeSessionMeta(size.getWidth(), size.getHeight(), clientResize);
                             encode(mediaCodec, streamer);
                         }