shadow: add support for frame acks

include/freerdp/server/shadow.h

@@ -29,6 +29,10 @@
 #include <freerdp/codec/color.h>
 #include <freerdp/codec/region.h>
 
+#include <winpr/crt.h>
+#include <winpr/synch.h>
+#include <winpr/collections.h>
+
 typedef struct rdp_shadow_client rdpShadowClient;
 typedef struct rdp_shadow_server rdpShadowServer;
 typedef struct rdp_shadow_screen rdpShadowScreen;

server/shadow/X11/x11_shadow.c

@@ -177,10 +177,9 @@ int x11_shadow_invalidate_region(x11ShadowSubsystem* subsystem, int x, int y, in
 	invalidRect.right = x + width;
 	invalidRect.bottom = y + height;
 
-	printf("x11_shadow_invalidate_region: x: %d y: %d width: %d height: %d\n",
+	EnterCriticalSection(&(surface->lock));
-			x, y, width, height);
-
 	region16_union_rect(&(surface->invalidRegion), &(surface->invalidRegion), &invalidRect);
+	LeaveCriticalSection(&(surface->lock));
 
 	return 1;
 }
@@ -208,8 +207,7 @@ int x11_shadow_surface_copy(x11ShadowSubsystem* subsystem)
 	width = extents->right - extents->left;
 	height = extents->bottom - extents->top;
 
-	printf("x11_shadow_surface_copy: x: %d y: %d width: %d height: %d\n",
+	XLockDisplay(subsystem->display);
-			x, y, width, height);
 
 	if (subsystem->use_xshm)
 	{
@@ -226,7 +224,7 @@ int x11_shadow_surface_copy(x11ShadowSubsystem* subsystem)
 	else
 	{
 		image = XGetImage(subsystem->display, subsystem->root_window,
-						x, y, width, height, AllPlanes, ZPixmap);
+				x, y, width, height, AllPlanes, ZPixmap);
 
 		freerdp_image_copy(surface->data, PIXEL_FORMAT_XRGB32,
 				surface->scanline, x, y, width, height,
@@ -238,34 +236,57 @@ int x11_shadow_surface_copy(x11ShadowSubsystem* subsystem)
 
 	x11_shadow_validate_region(subsystem, x, y, width, height);
 
+	XUnlockDisplay(subsystem->display);
+
 	return 1;
 }
 
-int x11_shadow_check_event(x11ShadowSubsystem* subsystem)
+void* x11_shadow_subsystem_thread(x11ShadowSubsystem* subsystem)
 {
+	DWORD status;
+	DWORD nCount;
 	XEvent xevent;
+	HANDLE events[32];
+	HANDLE StopEvent;
 	int x, y, width, height;
 	XDamageNotifyEvent* notify;
 
-	while (XPending(subsystem->display) > 0)
+	StopEvent = subsystem->server->StopEvent;
+
+	nCount = 0;
+	events[nCount++] = StopEvent;
+	events[nCount++] = subsystem->event;
+
+	while (1)
 	{
-		ZeroMemory(&xevent, sizeof(xevent));
+		status = WaitForMultipleObjects(nCount, events, FALSE, INFINITE);
-		XNextEvent(subsystem->display, &xevent);
 
-		if (xevent.type == subsystem->xdamage_notify_event)
+		if (WaitForSingleObject(StopEvent, 0) == WAIT_OBJECT_0)
 		{
-			notify = (XDamageNotifyEvent*) &xevent;
+			break;
+		}
 
-			x = notify->area.x;
+		while (XPending(subsystem->display))
-			y = notify->area.y;
+		{
-			width = notify->area.width;
+			ZeroMemory(&xevent, sizeof(xevent));
-			height = notify->area.height;
+			XNextEvent(subsystem->display, &xevent);
 
-			x11_shadow_invalidate_region(subsystem, x, y, width, height);
+			if (xevent.type == subsystem->xdamage_notify_event)
+			{
+				notify = (XDamageNotifyEvent*) &xevent;
+
+				x = notify->area.x;
+				y = notify->area.y;
+				width = notify->area.width;
+				height = notify->area.height;
+
+				x11_shadow_invalidate_region(subsystem, x, y, width, height);
+			}
 		}
 	}
 
-	return 1;
+	ExitThread(0);
+	return NULL;
 }
 
 int x11_shadow_cursor_init(x11ShadowSubsystem* subsystem)
@@ -275,10 +296,7 @@ int x11_shadow_cursor_init(x11ShadowSubsystem* subsystem)
 	int error;
 
 	if (!XFixesQueryExtension(subsystem->display, &event, &error))
-	{
-		fprintf(stderr, "XFixesQueryExtension failed\n");
 		return -1;
-	}
 
 	subsystem->xfixes_notify_event = event + XFixesCursorNotify;
 
@@ -304,10 +322,7 @@ int x11_shadow_xdamage_init(x11ShadowSubsystem* subsystem)
 		return -1;
 
 	if (major < 1)
-	{
-		fprintf(stderr, "XDamageQueryVersion failed: major:%d minor:%d\n", major, minor);
 		return -1;
-	}
 
 	subsystem->xdamage_notify_event = damage_event + XDamageNotify;
 	subsystem->xdamage = XDamageCreate(subsystem->display, subsystem->root_window, XDamageReportDeltaRectangles);
@@ -349,10 +364,7 @@ int x11_shadow_xshm_init(x11ShadowSubsystem* server)
 		return -1;
 
 	if (!pixmaps)
-	{
-		fprintf(stderr, "XShmQueryVersion failed\n");
 		return -1;
-	}
 
 	server->fb_shm_info.shmid = -1;
 	server->fb_shm_info.shmaddr = (char*) -1;
@@ -417,7 +429,6 @@ int x11_shadow_subsystem_init(x11ShadowSubsystem* subsystem)
 	XPixmapFormatValues* pfs;
 
 	/**
-	 * Recent X11 servers drop support for shared pixmaps
 	 * To see if your X11 server supports shared pixmaps, use:
 	 * xdpyinfo -ext MIT-SHM | grep "shared pixmaps"
 	 */
@@ -430,9 +441,7 @@ int x11_shadow_subsystem_init(x11ShadowSubsystem* subsystem)
 	}
 
 	if (!XInitThreads())
-	{
+		return -1;
-		fprintf(stderr, "warning: XInitThreads() failure\n");
-	}
 
 	subsystem->display = XOpenDisplay(NULL);
 

server/shadow/X11/x11_shadow.h

@@ -87,8 +87,8 @@ struct x11_shadow_subsystem
 extern "C" {
 #endif
 
-int x11_shadow_check_event(x11ShadowSubsystem* subsystem);
 int x11_shadow_surface_copy(x11ShadowSubsystem* subsystem);
+void* x11_shadow_subsystem_thread(x11ShadowSubsystem* subsystem);
 
 void x11_shadow_input_synchronize_event(x11ShadowSubsystem* subsystem, UINT32 flags);
 void x11_shadow_input_keyboard_event(x11ShadowSubsystem* subsystem, UINT16 flags, UINT16 code);

server/shadow/shadow.c

@@ -31,6 +31,7 @@ void* shadow_server_thread(rdpShadowServer* server)
 {
 	DWORD status;
 	DWORD nCount;
+	HANDLE thread;
 	HANDLE events[32];
 	HANDLE StopEvent;
 	freerdp_listener* listener;
@@ -38,6 +39,10 @@ void* shadow_server_thread(rdpShadowServer* server)
 	listener = server->listener;
 	StopEvent = server->StopEvent;
 
+	thread = CreateThread(NULL, 0,
+			(LPTHREAD_START_ROUTINE) x11_shadow_subsystem_thread,
+			(void*) server->subsystem, 0, NULL);
+
 	while (1)
 	{
 		nCount = 0;

server/shadow/shadow_client.c

@@ -25,6 +25,7 @@
 #include <winpr/path.h>
 #include <winpr/synch.h>
 #include <winpr/thread.h>
+#include <winpr/sysinfo.h>
 
 #include <winpr/tools/makecert.h>
 
@@ -32,11 +33,19 @@
 
 void shadow_client_context_new(freerdp_peer* peer, rdpShadowClient* client)
 {
+	rdpSettings* settings;
 	rdpShadowServer* server;
 
 	server = (rdpShadowServer*) peer->ContextExtra;
 	client->server = server;
 
+	settings = peer->settings;
+	settings->ColorDepth = 32;
+	settings->RemoteFxCodec = TRUE;
+	settings->BitmapCacheV3Enabled = TRUE;
+	settings->FrameMarkerCommandEnabled = TRUE;
+	settings->SurfaceFrameMarkerEnabled = TRUE;
+
 	client->StopEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
 }
 
@@ -78,6 +87,40 @@ BOOL shadow_client_activate(freerdp_peer* peer)
 	return TRUE;
 }
 
+void shadow_client_surface_frame_acknowledge(rdpShadowClient* client, UINT32 frameId)
+{
+	SURFACE_FRAME* frame;
+	wListDictionary* frameList;
+
+	frameList = client->server->encoder->frameList;
+	frame = (SURFACE_FRAME*) ListDictionary_GetItemValue(frameList, (void*) (size_t) frameId);
+
+	if (frame)
+	{
+		ListDictionary_Remove(frameList, (void*) (size_t) frameId);
+		free(frame);
+	}
+}
+
+void shadow_client_suppress_output(rdpShadowClient* client, BYTE allow, RECTANGLE_16* area)
+{
+
+}
+
+int shadow_client_send_surface_frame_marker(rdpShadowClient* client, UINT32 action, UINT32 id)
+{
+	SURFACE_FRAME_MARKER surfaceFrameMarker;
+	rdpContext* context = (rdpContext*) client;
+	rdpUpdate* update = context->update;
+
+	surfaceFrameMarker.frameAction = action;
+	surfaceFrameMarker.frameId = id;
+
+	IFCALL(update->SurfaceFrameMarker, context, &surfaceFrameMarker);
+
+	return 1;
+}
+
 int shadow_client_send_surface_bits(rdpShadowClient* client)
 {
 	int i;
@@ -89,6 +132,7 @@ int shadow_client_send_surface_bits(rdpShadowClient* client)
 	int nSrcStep;
 	BYTE* pSrcData;
 	int numMessages;
+	UINT32 frameId = 0;
 	rdpUpdate* update;
 	rdpContext* context;
 	rdpSettings* settings;
@@ -126,6 +170,12 @@ int shadow_client_send_surface_bits(rdpShadowClient* client)
 	pSrcData = surface->data;
 	nSrcStep = surface->scanline;
 
+	if (encoder->frameAck)
+	{
+		frameId = (UINT32) shadow_encoder_create_frame_id(encoder);
+		shadow_client_send_surface_frame_marker(client, SURFACECMD_FRAMEACTION_BEGIN, frameId);
+	}
+
 	if (settings->RemoteFxCodec)
 	{
 		RFX_RECT rect;
@@ -205,6 +255,11 @@ int shadow_client_send_surface_bits(rdpShadowClient* client)
 
 	region16_clear(&(surface->invalidRegion));
 
+	if (encoder->frameAck)
+	{
+		shadow_client_send_surface_frame_marker(client, SURFACECMD_FRAMEACTION_END, frameId);
+	}
+
 	return 0;
 }
 
@@ -256,19 +311,25 @@ int shadow_generate_certificate(rdpSettings* settings)
 
 void* shadow_client_thread(rdpShadowClient* client)
 {
+	int fps;
 	DWORD status;
 	DWORD nCount;
+	UINT64 cTime;
+	DWORD dwTimeout;
+	DWORD dwInterval;
+	UINT64 frameTime;
 	HANDLE events[32];
 	HANDLE StopEvent;
 	HANDLE ClientEvent;
-	HANDLE SubsystemEvent;
 	freerdp_peer* peer;
 	rdpSettings* settings;
 	rdpShadowServer* server;
 	rdpShadowSurface* surface;
+	rdpShadowEncoder* encoder;
 	rdpShadowSubsystem* subsystem;
 
 	server = client->server;
+	encoder = server->encoder;
 	surface = server->surface;
 	subsystem = server->subsystem;
 
@@ -292,18 +353,27 @@ void* shadow_client_thread(rdpShadowClient* client)
 
 	peer->Initialize(peer);
 
+	peer->update->SurfaceFrameAcknowledge = (pSurfaceFrameAcknowledge)
+			shadow_client_surface_frame_acknowledge;
+	peer->update->SuppressOutput = (pSuppressOutput) shadow_client_suppress_output;
+
+	fps = 16;
+	dwInterval = 1000 / fps;
+	frameTime = GetTickCount64() + dwInterval;
+
 	StopEvent = client->StopEvent;
 	ClientEvent = peer->GetEventHandle(peer);
-	SubsystemEvent = subsystem->event;
 
 	while (1)
 	{
 		nCount = 0;
 		events[nCount++] = StopEvent;
 		events[nCount++] = ClientEvent;
-		events[nCount++] = SubsystemEvent;
 
-		status = WaitForMultipleObjects(nCount, events, FALSE, 250);
+		cTime = GetTickCount64();
+		dwTimeout = (cTime > frameTime) ? 0 : frameTime - cTime;
+
+		status = WaitForMultipleObjects(nCount, events, FALSE, dwTimeout);
 
 		if (WaitForSingleObject(client->StopEvent, 0) == WAIT_OBJECT_0)
 		{
@@ -319,16 +389,20 @@ void* shadow_client_thread(rdpShadowClient* client)
 			}
 		}
 
-		if (WaitForSingleObject(SubsystemEvent, 0) == WAIT_OBJECT_0)
+		if ((status == WAIT_TIMEOUT) || (GetTickCount64() > frameTime))
 		{
-			x11_shadow_check_event((x11ShadowSubsystem*) subsystem);
+			if (client->activated)
-		}
+			{
+				EnterCriticalSection(&(surface->lock));
+				x11_shadow_surface_copy((x11ShadowSubsystem*) subsystem);
+				shadow_client_send_surface_bits(client);
+				region16_clear(&(surface->invalidRegion));
+				LeaveCriticalSection(&(surface->lock));
+			}
 
-		if (client->activated)
+			fps = encoder->fps;
-		{
+			dwInterval = 1000 / fps;
-			x11_shadow_surface_copy((x11ShadowSubsystem*) subsystem);
+			frameTime += dwInterval;
-			shadow_client_send_surface_bits(client);
-			region16_clear(&(surface->invalidRegion));
 		}
 	}
 

server/shadow/shadow_encoder.c

@@ -24,6 +24,40 @@
 
 #include "shadow_encoder.h"
 
+int shadow_encoder_create_frame_id(rdpShadowEncoder* encoder)
+{
+	UINT32 frameId;
+	int inFlightFrames;
+	SURFACE_FRAME* frame;
+
+	inFlightFrames = ListDictionary_Count(encoder->frameList);
+
+	if (inFlightFrames > encoder->frameAck)
+	{
+		encoder->fps = (100 / (inFlightFrames + 1) * encoder->maxFps) / 100;
+	}
+	else
+	{
+		encoder->fps += 2;
+
+		if (encoder->fps > encoder->maxFps)
+			encoder->fps = encoder->maxFps;
+	}
+
+	if (encoder->fps < 1)
+		encoder->fps = 1;
+
+	frame = (SURFACE_FRAME*) malloc(sizeof(SURFACE_FRAME));
+
+	if (!frame)
+		return -1;
+
+	frameId = frame->frameId = ++encoder->frameId;
+	ListDictionary_Add(encoder->frameList, (void*) (size_t) frame->frameId, frame);
+
+	return (int) frame->frameId;
+}
+
 int shadow_encoder_grid_init(rdpShadowEncoder* encoder)
 {
 	int i, j, k;
@@ -129,6 +163,12 @@ rdpShadowEncoder* shadow_encoder_new(rdpShadowServer* server)
 
 	shadow_encoder_grid_init(encoder);
 
+	encoder->fps = 10;
+	encoder->maxFps = 32;
+	encoder->frameId = 0;
+	encoder->frameAck = TRUE;
+	encoder->frameList = ListDictionary_New(TRUE);
+
 	return encoder;
 }
 
@@ -150,5 +190,7 @@ void shadow_encoder_free(rdpShadowEncoder* encoder)
 
 	shadow_encoder_grid_uninit(encoder);
 
+	ListDictionary_Free(encoder->frameList);
+
 	free(encoder);
 }

server/shadow/shadow_encoder.h

@@ -57,12 +57,20 @@ struct rdp_shadow_encoder
 	wStream* bs;
 	wStream* bts;
 	BITMAP_PLANAR_CONTEXT* planar;
+
+	int fps;
+	int maxFps;
+	BOOL frameAck;
+	UINT32 frameId;
+	wListDictionary* frameList;
 };
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+int shadow_encoder_create_frame_id(rdpShadowEncoder* encoder);
+
 rdpShadowEncoder* shadow_encoder_new(rdpShadowServer* server);
 void shadow_encoder_free(rdpShadowEncoder* encoder);
 

server/shadow/shadow_surface.c

@@ -46,6 +46,9 @@ rdpShadowSurface* shadow_surface_new(rdpShadowServer* server, int width, int hei
 
 	ZeroMemory(surface->data, surface->scanline * surface->height);
 
+	if (!InitializeCriticalSectionAndSpinCount(&(surface->lock), 4000))
+		return NULL;
+
 	region16_init(&(surface->invalidRegion));
 
 	return surface;
@@ -58,6 +61,8 @@ void shadow_surface_free(rdpShadowSurface* surface)
 
 	free(surface->data);
 
+	DeleteCriticalSection(&(surface->lock));
+
 	region16_uninit(&(surface->invalidRegion));
 
 	free(surface);

server/shadow/shadow_surface.h

@@ -21,6 +21,9 @@
 
 #include <freerdp/server/shadow.h>
 
+#include <winpr/crt.h>
+#include <winpr/synch.h>
+
 struct rdp_shadow_surface
 {
 	rdpShadowServer* server;
@@ -30,6 +33,7 @@ struct rdp_shadow_surface
 	int scanline;
 	BYTE* data;
 
+	CRITICAL_SECTION lock;
 	REGION16 invalidRegion;
 };