From 72c497e61df26b12b97c60e24ae92356107d65bb Mon Sep 17 00:00:00 2001 From: Andri Yngvason Date: Sun, 10 Apr 2022 16:55:54 +0000 Subject: [PATCH] vnc: Re-implement libvncclient's SetFormatAndEncodings Otherwise, we can't prioritise open-h264 --- meson.build | 1 + src/vnc.c | 4 +- src/vnc_encodings.c | 181 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 src/vnc_encodings.c diff --git a/meson.build b/meson.build index 8d356a0..9b24cfd 100644 --- a/meson.build +++ b/meson.build @@ -73,6 +73,7 @@ sources = [ 'src/pointer.c', 'src/keyboard.c', 'src/vnc.c', + 'src/vnc_encodings.c', 'src/strlcpy.c', 'src/evdev-to-qnum.c', 'src/pixels.c', diff --git a/src/vnc.c b/src/vnc.c index 9c80ffe..5d8ce82 100644 --- a/src/vnc.c +++ b/src/vnc.c @@ -32,6 +32,8 @@ extern const unsigned short code_map_linux_to_qnum[]; extern const unsigned int code_map_linux_to_qnum_len; +rfbBool vnc_client_set_format_and_encodings(rfbClient* client); + static rfbBool vnc_client_alloc_fb(rfbClient* client) { struct vnc_client* self = rfbClientGetClientData(client, NULL); @@ -190,7 +192,7 @@ int vnc_client_connect(struct vnc_client* self, const char* address, int port) if (!client->MallocFrameBuffer(client)) return -1; - if (!SetFormatAndEncodings(client)) + if (!vnc_client_set_format_and_encodings(client)) return -1; if (client->updateRect.x < 0) { diff --git a/src/vnc_encodings.c b/src/vnc_encodings.c new file mode 100644 index 0000000..d6e4853 --- /dev/null +++ b/src/vnc_encodings.c @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2022 Andri Yngvason. All Rights Reserved. + * Copyright (C) 2000-2002 Constantin Kaplinsky. All Rights Reserved. + * Copyright (C) 2000 Tridia Corporation. All Rights Reserved. + * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ + +#include +#include + +rfbBool vnc_client_set_format_and_encodings(rfbClient* client) +{ + assert(client->appData.encodingsString); + + rfbSetPixelFormatMsg spf; + union { + char bytes[sz_rfbSetEncodingsMsg + MAX_ENCODINGS*4]; + rfbSetEncodingsMsg msg; + } buf; + + rfbSetEncodingsMsg *se = &buf.msg; + uint32_t *encs = (uint32_t *)(&buf.bytes[sz_rfbSetEncodingsMsg]); + int len = 0; + rfbBool requestCompressLevel = FALSE; + rfbBool requestQualityLevel = FALSE; + rfbBool requestLastRectEncoding = FALSE; + + if (!SupportsClient2Server(client, rfbSetPixelFormat)) + return TRUE; + + spf.type = rfbSetPixelFormat; + spf.pad1 = 0; + spf.pad2 = 0; + spf.format = client->format; + spf.format.redMax = rfbClientSwap16IfLE(spf.format.redMax); + spf.format.greenMax = rfbClientSwap16IfLE(spf.format.greenMax); + spf.format.blueMax = rfbClientSwap16IfLE(spf.format.blueMax); + + if (!WriteToRFBServer(client, (char *)&spf, sz_rfbSetPixelFormatMsg)) + return FALSE; + + + if (!SupportsClient2Server(client, rfbSetEncodings)) + return TRUE; + + se->type = rfbSetEncodings; + se->pad = 0; + se->nEncodings = 0; + + const char *encStr = client->appData.encodingsString; + int encStrLen; + do { + const char *nextEncStr = strchr(encStr, ','); + if (nextEncStr) { + encStrLen = nextEncStr - encStr; + nextEncStr++; + } else { + encStrLen = strlen(encStr); + } + + if (strncasecmp(encStr,"raw",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRaw); + } else if (strncasecmp(encStr,"copyrect",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCopyRect); +#ifdef LIBVNCSERVER_HAVE_LIBZ +#ifdef LIBVNCSERVER_HAVE_LIBJPEG + } else if (strncasecmp(encStr,"tight",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingTight); + requestLastRectEncoding = TRUE; + if (client->appData.compressLevel >= 0 && client->appData.compressLevel <= 9) + requestCompressLevel = TRUE; + if (client->appData.enableJPEG) + requestQualityLevel = TRUE; +#endif +#endif + } else if (strncasecmp(encStr,"hextile",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingHextile); +#ifdef LIBVNCSERVER_HAVE_LIBZ + } else if (strncasecmp(encStr,"zlib",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZlib); + if (client->appData.compressLevel >= 0 && client->appData.compressLevel <= 9) + requestCompressLevel = TRUE; + } else if (strncasecmp(encStr,"zlibhex",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZlibHex); + if (client->appData.compressLevel >= 0 && client->appData.compressLevel <= 9) + requestCompressLevel = TRUE; + } else if (strncasecmp(encStr,"trle",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingTRLE); + } else if (strncasecmp(encStr,"zrle",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZRLE); + } else if (strncasecmp(encStr,"zywrle",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingZYWRLE); + requestQualityLevel = TRUE; +#endif + } else if ((strncasecmp(encStr,"ultra",encStrLen) == 0) || (strncasecmp(encStr,"ultrazip",encStrLen) == 0)) { + /* There are 2 encodings used in 'ultra' */ + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingUltra); + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingUltraZip); + } else if (strncasecmp(encStr,"corre",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingCoRRE); + } else if (strncasecmp(encStr,"rre",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRRE); + } else if (strncasecmp(encStr,"open-h264",encStrLen) == 0) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(50); + } else { + rfbClientLog("Unknown encoding '%.*s'\n",encStrLen,encStr); + } + + encStr = nextEncStr; + } while (encStr && se->nEncodings < MAX_ENCODINGS); + + if (se->nEncodings < MAX_ENCODINGS && requestCompressLevel) { + encs[se->nEncodings++] = rfbClientSwap32IfLE(client->appData.compressLevel + + rfbEncodingCompressLevel0); + } + + if (se->nEncodings < MAX_ENCODINGS && requestQualityLevel) { + if (client->appData.qualityLevel < 0 || client->appData.qualityLevel > 9) + client->appData.qualityLevel = 5; + encs[se->nEncodings++] = rfbClientSwap32IfLE(client->appData.qualityLevel + + rfbEncodingQualityLevel0); + } + + /* Remote Cursor Support (local to viewer) */ + if (client->appData.useRemoteCursor) { + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingXCursor); + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingRichCursor); + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingPointerPos); + } + + /* Keyboard State Encodings */ + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingKeyboardLedState); + + /* New Frame Buffer Size */ + if (se->nEncodings < MAX_ENCODINGS && client->canHandleNewFBSize) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingNewFBSize); + + /* Last Rect */ + if (se->nEncodings < MAX_ENCODINGS && requestLastRectEncoding) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingLastRect); + + /* Server Capabilities */ + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingSupportedMessages); + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingSupportedEncodings); + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingServerIdentity); + + /* xvp */ + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingXvp); + + if (se->nEncodings < MAX_ENCODINGS) + encs[se->nEncodings++] = rfbClientSwap32IfLE(rfbEncodingQemuExtendedKeyEvent); + + len = sz_rfbSetEncodingsMsg + se->nEncodings * 4; + + se->nEncodings = rfbClientSwap16IfLE(se->nEncodings); + + return WriteToRFBServer(client, buf.bytes, len); +}