From 8901ff5e582ccf318efbcf3f71d71d7aa4eecb83 Mon Sep 17 00:00:00 2001 From: PouleyKetchoupp Date: Mon, 29 Jul 2019 12:33:38 +0200 Subject: [PATCH 001/113] Update transforms after timers update to make sure changes are made before render Fixes #29952 (cherry picked from commit c6b82f108ad4deb1a1fc87eea905650127dd686d) --- scene/main/scene_tree.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scene/main/scene_tree.cpp b/scene/main/scene_tree.cpp index 7bdfecccc27..de182c1da3c 100644 --- a/scene/main/scene_tree.cpp +++ b/scene/main/scene_tree.cpp @@ -562,6 +562,8 @@ bool SceneTree::idle(float p_time) { E = N; } + flush_transform_notifications(); //additional transforms after timers update + _call_idle_callbacks(); #ifdef TOOLS_ENABLED From eb29bd988c182b71b5142b2f743dc2f3d78c48a8 Mon Sep 17 00:00:00 2001 From: Fabio Alessandrelli Date: Sun, 30 Jun 2019 17:31:51 +0200 Subject: [PATCH 002/113] Update miniupnpc library to latest master (cherry picked from commit 78907d91f140dd047a2eebb2736ed1762897003d) --- modules/upnp/SCsub | 3 +- thirdparty/README.md | 6 +- thirdparty/miniupnpc/LICENSE | 4 +- thirdparty/miniupnpc/miniupnpc.def | 45 --- thirdparty/miniupnpc/miniupnpc.h | 153 -------- .../miniupnpc/{ => miniupnpc}/codelength.h | 0 .../{ => miniupnpc}/connecthostport.c | 62 ++-- .../{ => miniupnpc}/connecthostport.h | 0 .../{ => miniupnpc}/igd_desc_parse.c | 0 .../{ => miniupnpc}/igd_desc_parse.h | 0 .../miniupnpc/{ => miniupnpc}/listdevices.c | 0 .../miniupnpc/{ => miniupnpc}/minisoap.c | 9 +- .../miniupnpc/{ => miniupnpc}/minisoap.h | 0 .../miniupnpc/{ => miniupnpc}/minissdpc.c | 101 +++-- .../miniupnpc/{ => miniupnpc}/minissdpc.h | 6 +- .../miniupnpc/{ => miniupnpc}/miniupnpc.c | 37 +- .../{ => miniupnpc}/miniupnpc_declspec.h | 0 .../{ => miniupnpc}/miniupnpc_socketdef.h | 7 + .../{ => miniupnpc}/miniupnpcmodule.c | 34 +- .../{ => miniupnpc}/miniupnpcstrings.h | 0 .../{ => miniupnpc}/miniupnpctypes.h | 0 .../miniupnpc/{ => miniupnpc}/miniwget.c | 9 +- .../{ => miniupnpc}/miniwget_private.h | 0 .../miniupnpc/{ => miniupnpc}/minixml.c | 0 .../miniupnpc/{ => miniupnpc}/minixml.h | 2 +- .../miniupnpc/{ => miniupnpc}/minixmlvalid.c | 0 .../{ => miniupnpc}/portlistingparse.c | 0 .../{ => miniupnpc}/portlistingparse.h | 0 .../miniupnpc/{ => miniupnpc}/receivedata.c | 0 .../miniupnpc/{ => miniupnpc}/receivedata.h | 0 thirdparty/miniupnpc/{ => miniupnpc}/upnpc.c | 4 +- .../miniupnpc/{ => miniupnpc}/upnpcommands.c | 63 ++-- thirdparty/miniupnpc/miniupnpc/upnpcommands.h | 6 +- .../miniupnpc/{ => miniupnpc}/upnpdev.c | 0 .../miniupnpc/{ => miniupnpc}/upnpdev.h | 0 .../miniupnpc/{ => miniupnpc}/upnperrors.c | 8 +- .../miniupnpc/{ => miniupnpc}/upnperrors.h | 0 .../{ => miniupnpc}/upnpreplyparse.c | 5 +- .../{ => miniupnpc}/upnpreplyparse.h | 0 thirdparty/miniupnpc/miniwget.h | 27 -- thirdparty/miniupnpc/upnpcommands.h | 348 ------------------ 41 files changed, 196 insertions(+), 743 deletions(-) delete mode 100644 thirdparty/miniupnpc/miniupnpc.def delete mode 100644 thirdparty/miniupnpc/miniupnpc.h rename thirdparty/miniupnpc/{ => miniupnpc}/codelength.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/connecthostport.c (86%) rename thirdparty/miniupnpc/{ => miniupnpc}/connecthostport.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/igd_desc_parse.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/igd_desc_parse.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/listdevices.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/minisoap.c (94%) rename thirdparty/miniupnpc/{ => miniupnpc}/minisoap.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/minissdpc.c (92%) rename thirdparty/miniupnpc/{ => miniupnpc}/minissdpc.h (91%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpc.c (97%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpc_declspec.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpc_socketdef.h (88%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpcmodule.c (95%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpcstrings.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniupnpctypes.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniwget.c (99%) rename thirdparty/miniupnpc/{ => miniupnpc}/miniwget_private.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/minixml.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/minixml.h (93%) rename thirdparty/miniupnpc/{ => miniupnpc}/minixmlvalid.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/portlistingparse.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/portlistingparse.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/receivedata.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/receivedata.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpc.c (99%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpcommands.c (97%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpdev.c (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpdev.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnperrors.c (90%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnperrors.h (100%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpreplyparse.c (97%) rename thirdparty/miniupnpc/{ => miniupnpc}/upnpreplyparse.h (100%) delete mode 100644 thirdparty/miniupnpc/miniwget.h delete mode 100644 thirdparty/miniupnpc/upnpcommands.h diff --git a/modules/upnp/SCsub b/modules/upnp/SCsub index 2b15f7aee2c..42b931095dd 100644 --- a/modules/upnp/SCsub +++ b/modules/upnp/SCsub @@ -23,10 +23,11 @@ if env['builtin_miniupnpc']: "portlistingparse.c", "upnpreplyparse.c", ] - thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources] + thirdparty_sources = [thirdparty_dir + "miniupnpc/" + file for file in thirdparty_sources] env_upnp.Append(CPPPATH=[thirdparty_dir]) env_upnp.Append(CPPFLAGS=["-DMINIUPNP_STATICLIB"]) + env_upnp.Append(CPPFLAGS=["-DMINIUPNPC_SET_SOCKET_TIMEOUT"]) env_thirdparty = env_upnp.Clone() env_thirdparty.disable_warnings() diff --git a/thirdparty/README.md b/thirdparty/README.md index 7bb84a3f317..702943e1bc0 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -297,9 +297,13 @@ File extracted from upstream release tarball `mbedtls-2.16.0-apache.tgz`: ## miniupnpc - Upstream: https://github.com/miniupnp/miniupnp/tree/master/miniupnpc -- Version: git (25615e0, 2018) +- Version: git (3cf6efa, 2019) - License: BSD-3-Clause +Extract only the `miniupnpc` folder inside `thirdparty/miniupnpc`. +Exclude all non `.c` and `.h` files, plus all files beginning with `test` +`minihttptestserver.c` and `wingenminiupnpcstrings.c`. + The only modified file is miniupnpcstrings.h, which was created for Godot (it is usually autogenerated by cmake). diff --git a/thirdparty/miniupnpc/LICENSE b/thirdparty/miniupnpc/LICENSE index 08167337041..39e0345f8a7 100644 --- a/thirdparty/miniupnpc/LICENSE +++ b/thirdparty/miniupnpc/LICENSE @@ -1,5 +1,5 @@ -MiniUPnPc -Copyright (c) 2005-2016, Thomas BERNARD +MiniUPnP Project +Copyright (c) 2005-2019, Thomas BERNARD All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/thirdparty/miniupnpc/miniupnpc.def b/thirdparty/miniupnpc/miniupnpc.def deleted file mode 100644 index 60e0bbe4232..00000000000 --- a/thirdparty/miniupnpc/miniupnpc.def +++ /dev/null @@ -1,45 +0,0 @@ -LIBRARY -; miniupnpc library - miniupnpc - -EXPORTS -; miniupnpc - upnpDiscover - freeUPNPDevlist - parserootdesc - UPNP_GetValidIGD - UPNP_GetIGDFromUrl - GetUPNPUrls - FreeUPNPUrls -; miniwget - miniwget - miniwget_getaddr -; upnpcommands - UPNP_GetTotalBytesSent - UPNP_GetTotalBytesReceived - UPNP_GetTotalPacketsSent - UPNP_GetTotalPacketsReceived - UPNP_GetStatusInfo - UPNP_GetConnectionTypeInfo - UPNP_GetExternalIPAddress - UPNP_GetLinkLayerMaxBitRates - UPNP_AddPortMapping - UPNP_AddAnyPortMapping - UPNP_DeletePortMapping - UPNP_DeletePortMappingRange - UPNP_GetPortMappingNumberOfEntries - UPNP_GetSpecificPortMappingEntry - UPNP_GetGenericPortMappingEntry - UPNP_GetListOfPortMappings - UPNP_AddPinhole - UPNP_CheckPinholeWorking - UPNP_UpdatePinhole - UPNP_GetPinholePackets - UPNP_DeletePinhole - UPNP_GetFirewallStatus - UPNP_GetOutboundPinholeTimeout -; upnperrors - strupnperror -; portlistingparse - ParsePortListing - FreePortListing diff --git a/thirdparty/miniupnpc/miniupnpc.h b/thirdparty/miniupnpc/miniupnpc.h deleted file mode 100644 index 8ddc282bd1e..00000000000 --- a/thirdparty/miniupnpc/miniupnpc.h +++ /dev/null @@ -1,153 +0,0 @@ -/* $Id: miniupnpc.h,v 1.53 2018/05/07 11:05:16 nanard Exp $ */ -/* vim: tabstop=4 shiftwidth=4 noexpandtab - * Project: miniupnp - * http://miniupnp.free.fr/ - * Author: Thomas Bernard - * Copyright (c) 2005-2018 Thomas Bernard - * This software is subjects to the conditions detailed - * in the LICENCE file provided within this distribution */ -#ifndef MINIUPNPC_H_INCLUDED -#define MINIUPNPC_H_INCLUDED - -#include "miniupnpc_declspec.h" -#include "igd_desc_parse.h" -#include "upnpdev.h" - -/* error codes : */ -#define UPNPDISCOVER_SUCCESS (0) -#define UPNPDISCOVER_UNKNOWN_ERROR (-1) -#define UPNPDISCOVER_SOCKET_ERROR (-101) -#define UPNPDISCOVER_MEMORY_ERROR (-102) - -/* versions : */ -#define MINIUPNPC_VERSION "2.1" -#define MINIUPNPC_API_VERSION 17 - -/* Source port: - Using "1" as an alias for 1900 for backwards compatibility - (presuming one would have used that for the "sameport" parameter) */ -#define UPNP_LOCAL_PORT_ANY 0 -#define UPNP_LOCAL_PORT_SAME 1 - -#ifdef __cplusplus -extern "C" { -#endif - -/* Structures definitions : */ -struct UPNParg { const char * elt; const char * val; }; - -char * -simpleUPnPcommand(int, const char *, const char *, - const char *, struct UPNParg *, - int *); - -/* upnpDiscover() - * discover UPnP devices on the network. - * The discovered devices are returned as a chained list. - * It is up to the caller to free the list with freeUPNPDevlist(). - * delay (in millisecond) is the maximum time for waiting any device - * response. - * If available, device list will be obtained from MiniSSDPd. - * Default path for minissdpd socket will be used if minissdpdsock argument - * is NULL. - * If multicastif is not NULL, it will be used instead of the default - * multicast interface for sending SSDP discover packets. - * If localport is set to UPNP_LOCAL_PORT_SAME(1) SSDP packets will be sent - * from the source port 1900 (same as destination port), if set to - * UPNP_LOCAL_PORT_ANY(0) system assign a source port, any other value will - * be attempted as the source port. - * "searchalltypes" parameter is useful when searching several types, - * if 0, the discovery will stop with the first type returning results. - * TTL should default to 2. */ -MINIUPNP_LIBSPEC struct UPNPDev * -upnpDiscover(int delay, const char * multicastif, - const char * minissdpdsock, int localport, - int ipv6, unsigned char ttl, - int * error); - -MINIUPNP_LIBSPEC struct UPNPDev * -upnpDiscoverAll(int delay, const char * multicastif, - const char * minissdpdsock, int localport, - int ipv6, unsigned char ttl, - int * error); - -MINIUPNP_LIBSPEC struct UPNPDev * -upnpDiscoverDevice(const char * device, int delay, const char * multicastif, - const char * minissdpdsock, int localport, - int ipv6, unsigned char ttl, - int * error); - -MINIUPNP_LIBSPEC struct UPNPDev * -upnpDiscoverDevices(const char * const deviceTypes[], - int delay, const char * multicastif, - const char * minissdpdsock, int localport, - int ipv6, unsigned char ttl, - int * error, - int searchalltypes); - -/* parserootdesc() : - * parse root XML description of a UPnP device and fill the IGDdatas - * structure. */ -MINIUPNP_LIBSPEC void parserootdesc(const char *, int, struct IGDdatas *); - -/* structure used to get fast access to urls - * controlURL: controlURL of the WANIPConnection - * ipcondescURL: url of the description of the WANIPConnection - * controlURL_CIF: controlURL of the WANCommonInterfaceConfig - * controlURL_6FC: controlURL of the WANIPv6FirewallControl - */ -struct UPNPUrls { - char * controlURL; - char * ipcondescURL; - char * controlURL_CIF; - char * controlURL_6FC; - char * rootdescURL; -}; - -/* UPNP_GetValidIGD() : - * return values : - * 0 = NO IGD found - * 1 = A valid connected IGD has been found - * 2 = A valid IGD has been found but it reported as - * not connected - * 3 = an UPnP device has been found but was not recognized as an IGD - * - * In any non zero return case, the urls and data structures - * passed as parameters are set. Donc forget to call FreeUPNPUrls(urls) to - * free allocated memory. - */ -MINIUPNP_LIBSPEC int -UPNP_GetValidIGD(struct UPNPDev * devlist, - struct UPNPUrls * urls, - struct IGDdatas * data, - char * lanaddr, int lanaddrlen); - -/* UPNP_GetIGDFromUrl() - * Used when skipping the discovery process. - * When succeding, urls, data, and lanaddr arguments are set. - * return value : - * 0 - Not ok - * 1 - OK */ -MINIUPNP_LIBSPEC int -UPNP_GetIGDFromUrl(const char * rootdescurl, - struct UPNPUrls * urls, - struct IGDdatas * data, - char * lanaddr, int lanaddrlen); - -MINIUPNP_LIBSPEC void -GetUPNPUrls(struct UPNPUrls *, struct IGDdatas *, - const char *, unsigned int); - -MINIUPNP_LIBSPEC void -FreeUPNPUrls(struct UPNPUrls *); - -/* return 0 or 1 */ -MINIUPNP_LIBSPEC int UPNPIGD_IsConnected(struct UPNPUrls *, struct IGDdatas *); - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/thirdparty/miniupnpc/codelength.h b/thirdparty/miniupnpc/miniupnpc/codelength.h similarity index 100% rename from thirdparty/miniupnpc/codelength.h rename to thirdparty/miniupnpc/miniupnpc/codelength.h diff --git a/thirdparty/miniupnpc/connecthostport.c b/thirdparty/miniupnpc/miniupnpc/connecthostport.c similarity index 86% rename from thirdparty/miniupnpc/connecthostport.c rename to thirdparty/miniupnpc/miniupnpc/connecthostport.c index ea6e4e59437..a59dc824372 100644 --- a/thirdparty/miniupnpc/connecthostport.c +++ b/thirdparty/miniupnpc/miniupnpc/connecthostport.c @@ -1,8 +1,8 @@ -/* $Id: connecthostport.c,v 1.15 2015/10/09 16:26:19 nanard Exp $ */ +/* $Id: connecthostport.c,v 1.21 2019/04/23 12:11:08 nanard Exp $ */ /* vim: tabstop=4 shiftwidth=4 noexpandtab * Project : miniupnp * Author : Thomas Bernard - * Copyright (c) 2010-2018 Thomas Bernard + * Copyright (c) 2010-2019 Thomas Bernard * This software is subject to the conditions detailed in the * LICENCE file provided in this distribution. */ @@ -41,13 +41,6 @@ #include #endif /* #else _WIN32 */ -/* definition of PRINT_SOCKET_ERROR */ -#ifdef _WIN32 -#define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); -#else -#define PRINT_SOCKET_ERROR(x) perror(x) -#endif - #if defined(__amigaos__) || defined(__amigaos4__) #define herror(A) printf("%s\n", A) #endif @@ -123,8 +116,22 @@ SOCKET connecthostport(const char * host, unsigned short port, int err; FD_ZERO(&wset); FD_SET(s, &wset); - if((n = select(s + 1, NULL, &wset, NULL, NULL)) == -1 && errno == EINTR) +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + timeout.tv_sec = 3; + timeout.tv_usec = 0; + n = select(s + 1, NULL, &wset, NULL, &timeout); +#else + n = select(s + 1, NULL, &wset, NULL, NULL); +#endif + if(n == -1 && errno == EINTR) continue; +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + if(n == 0) { + errno = ETIMEDOUT; + n = -1; + break; + } +#endif /*len = 0;*/ /*n = getpeername(s, NULL, &len);*/ len = sizeof(err); @@ -163,7 +170,7 @@ SOCKET connecthostport(const char * host, unsigned short port, for(i = 0, j = 1; host[j] && (host[j] != ']') && i < MAXHOSTNAMELEN; i++, j++) { tmp_host[i] = host[j]; - if(0 == memcmp(host+j, "%25", 3)) /* %25 is just url encoding for '%' */ + if(0 == strncmp(host+j, "%25", 3)) /* %25 is just url encoding for '%' */ j+=2; /* skip "25" */ } tmp_host[i] = '\0'; @@ -183,9 +190,11 @@ SOCKET connecthostport(const char * host, unsigned short port, #endif return INVALID_SOCKET; } - s = -1; + s = INVALID_SOCKET; for(p = ai; p; p = p->ai_next) { + if(!ISINVALID(s)) + closesocket(s); s = socket(p->ai_family, p->ai_socktype, p->ai_protocol); if(ISINVALID(s)) continue; @@ -208,7 +217,7 @@ SOCKET connecthostport(const char * host, unsigned short port, PRINT_SOCKET_ERROR("setsockopt"); } #endif /* #ifdef MINIUPNPC_SET_SOCKET_TIMEOUT */ - n = connect(s, p->ai_addr, p->ai_addrlen); + n = connect(s, p->ai_addr, MSC_CAST_INT p->ai_addrlen); #ifdef MINIUPNPC_IGNORE_EINTR /* EINTR The system call was interrupted by a signal that was caught * EINPROGRESS The socket is nonblocking and the connection cannot @@ -220,8 +229,22 @@ SOCKET connecthostport(const char * host, unsigned short port, int err; FD_ZERO(&wset); FD_SET(s, &wset); - if((n = select(s + 1, NULL, &wset, NULL, NULL)) == -1 && errno == EINTR) +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + timeout.tv_sec = 3; + timeout.tv_usec = 0; + n = select(s + 1, NULL, &wset, NULL, &timeout); +#else + n = select(s + 1, NULL, &wset, NULL, NULL); +#endif + if(n == -1 && errno == EINTR) continue; +#ifdef MINIUPNPC_SET_SOCKET_TIMEOUT + if(n == 0) { + errno = ETIMEDOUT; + n = -1; + break; + } +#endif /*len = 0;*/ /*n = getpeername(s, NULL, &len);*/ len = sizeof(err); @@ -237,15 +260,8 @@ SOCKET connecthostport(const char * host, unsigned short port, } } #endif /* #ifdef MINIUPNPC_IGNORE_EINTR */ - if(n < 0) - { - closesocket(s); - continue; - } - else - { + if(n >= 0) /* connect() was successful */ break; - } } freeaddrinfo(ai); if(ISINVALID(s)) @@ -256,9 +272,9 @@ SOCKET connecthostport(const char * host, unsigned short port, if(n < 0) { PRINT_SOCKET_ERROR("connect"); + closesocket(s); return INVALID_SOCKET; } #endif /* #ifdef USE_GETHOSTBYNAME */ return s; } - diff --git a/thirdparty/miniupnpc/connecthostport.h b/thirdparty/miniupnpc/miniupnpc/connecthostport.h similarity index 100% rename from thirdparty/miniupnpc/connecthostport.h rename to thirdparty/miniupnpc/miniupnpc/connecthostport.h diff --git a/thirdparty/miniupnpc/igd_desc_parse.c b/thirdparty/miniupnpc/miniupnpc/igd_desc_parse.c similarity index 100% rename from thirdparty/miniupnpc/igd_desc_parse.c rename to thirdparty/miniupnpc/miniupnpc/igd_desc_parse.c diff --git a/thirdparty/miniupnpc/igd_desc_parse.h b/thirdparty/miniupnpc/miniupnpc/igd_desc_parse.h similarity index 100% rename from thirdparty/miniupnpc/igd_desc_parse.h rename to thirdparty/miniupnpc/miniupnpc/igd_desc_parse.h diff --git a/thirdparty/miniupnpc/listdevices.c b/thirdparty/miniupnpc/miniupnpc/listdevices.c similarity index 100% rename from thirdparty/miniupnpc/listdevices.c rename to thirdparty/miniupnpc/miniupnpc/listdevices.c diff --git a/thirdparty/miniupnpc/minisoap.c b/thirdparty/miniupnpc/miniupnpc/minisoap.c similarity index 94% rename from thirdparty/miniupnpc/minisoap.c rename to thirdparty/miniupnpc/miniupnpc/minisoap.c index 520c9302e8d..f92b36ce891 100644 --- a/thirdparty/miniupnpc/minisoap.c +++ b/thirdparty/miniupnpc/miniupnpc/minisoap.c @@ -25,12 +25,6 @@ /* only for malloc */ #include -#ifdef _WIN32 -#define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); -#else -#define PRINT_SOCKET_ERROR(x) perror(x) -#endif - /* httpWrite sends the headers and the body to the socket * and returns the number of bytes sent */ static int @@ -79,11 +73,10 @@ int soapPostSubmit(SOCKET fd, const char * body, const char * httpversion) { - int bodysize; char headerbuf[512]; int headerssize; char portstr[8]; - bodysize = (int)strlen(body); + int bodysize = (int)strlen(body); /* We are not using keep-alive HTTP connections. * HTTP/1.1 needs the header Connection: close to do that. * This is the default with HTTP/1.0 diff --git a/thirdparty/miniupnpc/minisoap.h b/thirdparty/miniupnpc/miniupnpc/minisoap.h similarity index 100% rename from thirdparty/miniupnpc/minisoap.h rename to thirdparty/miniupnpc/miniupnpc/minisoap.h diff --git a/thirdparty/miniupnpc/minissdpc.c b/thirdparty/miniupnpc/miniupnpc/minissdpc.c similarity index 92% rename from thirdparty/miniupnpc/minissdpc.c rename to thirdparty/miniupnpc/miniupnpc/minissdpc.c index 1d29b4ba5b5..29f8110155e 100644 --- a/thirdparty/miniupnpc/minissdpc.c +++ b/thirdparty/miniupnpc/miniupnpc/minissdpc.c @@ -1,9 +1,9 @@ -/* $Id: minissdpc.c,v 1.32 2016/10/07 09:04:36 nanard Exp $ */ +/* $Id: minissdpc.c,v 1.40 2019/04/23 12:12:55 nanard Exp $ */ /* vim: tabstop=4 shiftwidth=4 noexpandtab * Project : miniupnp * Web : http://miniupnp.free.fr/ * Author : Thomas BERNARD - * copyright (c) 2005-2018 Thomas Bernard + * copyright (c) 2005-2019 Thomas Bernard * This software is subjet to the conditions detailed in the * provided LICENCE file. */ /*#include */ @@ -381,6 +381,7 @@ free_tmp_and_return: * the last 4 arguments are filled during the parsing : * - location/locationsize : "location:" field of the SSDP reply packet * - st/stsize : "st:" field of the SSDP reply packet. + * - usn/usnsize : "usn:" filed of the SSDP reply packet * The strings are NOT null terminated */ static void parseMSEARCHReply(const char * reply, int size, @@ -418,17 +419,17 @@ parseMSEARCHReply(const char * reply, int size, putchar('\n');*/ /* skip the colon and white spaces */ do { b++; } while(reply[b]==' '); - if(0==strncasecmp(reply+a, "location", 8)) + if(0==strncasecmp(reply+a, "location:", 9)) { *location = reply+b; *locationsize = i-b; } - else if(0==strncasecmp(reply+a, "st", 2)) + else if(0==strncasecmp(reply+a, "st:", 3)) { *st = reply+b; *stsize = i-b; } - else if(0==strncasecmp(reply+a, "usn", 3)) + else if(0==strncasecmp(reply+a, "usn:", 4)) { *usn = reply+b; *usnsize = i-b; @@ -471,7 +472,7 @@ ssdpDiscoverDevices(const char * const deviceTypes[], int searchalltypes) { struct UPNPDev * tmp; - struct UPNPDev * devlist = 0; + struct UPNPDev * devlist = NULL; unsigned int scope_id = 0; int opt = 1; static const char MSearchMsgFmt[] = @@ -491,7 +492,7 @@ ssdpDiscoverDevices(const char * const deviceTypes[], struct sockaddr_storage sockudp_w; #else int rv; - struct addrinfo hints, *servinfo, *p; + struct addrinfo hints, *servinfo; #endif #ifdef _WIN32 unsigned long _ttl = (unsigned long)ttl; @@ -545,51 +546,42 @@ ssdpDiscoverDevices(const char * const deviceTypes[], destAddr.sin_addr.s_addr = inet_addr("223.255.255.255"); destAddr.sin_port = 0; if (GetBestInterfaceEx((struct sockaddr *)&destAddr, &ifbestidx) == NO_ERROR) { - DWORD dwSize = 0; - DWORD dwRetVal = 0; - unsigned int i = 0; - ULONG flags = GAA_FLAG_INCLUDE_PREFIX; - ULONG family = AF_INET; - LPVOID lpMsgBuf = NULL; + DWORD dwRetVal = NO_ERROR; PIP_ADAPTER_ADDRESSES pAddresses = NULL; - ULONG outBufLen = 0; - ULONG Iterations = 0; + ULONG outBufLen = 15360; + int Iterations; PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; PIP_ADAPTER_UNICAST_ADDRESS pUnicast = NULL; - PIP_ADAPTER_ANYCAST_ADDRESS pAnycast = NULL; - PIP_ADAPTER_MULTICAST_ADDRESS pMulticast = NULL; - IP_ADAPTER_DNS_SERVER_ADDRESS *pDnServer = NULL; - IP_ADAPTER_PREFIX *pPrefix = NULL; - outBufLen = 15360; - do { + for (Iterations = 0; Iterations < 3; Iterations++) { pAddresses = (IP_ADAPTER_ADDRESSES *) HeapAlloc(GetProcessHeap(), 0, outBufLen); if (pAddresses == NULL) { break; } - dwRetVal = GetAdaptersAddresses(family, flags, NULL, pAddresses, &outBufLen); + dwRetVal = GetAdaptersAddresses(AF_INET, GAA_FLAG_INCLUDE_PREFIX, NULL, pAddresses, &outBufLen); - if (dwRetVal == ERROR_BUFFER_OVERFLOW) { - HeapFree(GetProcessHeap(), 0, pAddresses); - pAddresses = NULL; - } else { + if (dwRetVal != ERROR_BUFFER_OVERFLOW) { break; } - Iterations++; - } while ((dwRetVal == ERROR_BUFFER_OVERFLOW) && (Iterations < 3)); + HeapFree(GetProcessHeap(), 0, pAddresses); + pAddresses = NULL; + } if (dwRetVal == NO_ERROR) { pCurrAddresses = pAddresses; while (pCurrAddresses) { #ifdef DEBUG + int i; + PIP_ADAPTER_MULTICAST_ADDRESS pMulticast = NULL; + PIP_ADAPTER_ANYCAST_ADDRESS pAnycast = NULL; + printf("\tIfIndex (IPv4 interface): %u\n", pCurrAddresses->IfIndex); printf("\tAdapter name: %s\n", pCurrAddresses->AdapterName); pUnicast = pCurrAddresses->FirstUnicastAddress; if (pUnicast != NULL) { for (i = 0; pUnicast != NULL; i++) { - IPAddr.S_un.S_addr = (u_long) pUnicast->Address; - printf("\tIP Address[%d]: \t%s\n", i, inet_ntoa(IPAddr) ); + printf("\tIP Address[%d]: \t%s\n", i, inet_ntoa(((PSOCKADDR_IN)pUnicast->Address.lpSockaddr)->sin_addr) ); pUnicast = pUnicast->Next; } printf("\tNumber of Unicast Addresses: %d\n", i); @@ -597,8 +589,7 @@ ssdpDiscoverDevices(const char * const deviceTypes[], pAnycast = pCurrAddresses->FirstAnycastAddress; if (pAnycast) { for (i = 0; pAnycast != NULL; i++) { - IPAddr.S_un.S_addr = (u_long) pAnyCast->Address; - printf("\tAnycast Address[%d]: \t%s\n", i, inet_ntoa(IPAddr) ); + printf("\tAnycast Address[%d]: \t%s\n", i, inet_ntoa(((PSOCKADDR_IN)pAnycast->Address.lpSockaddr)->sin_addr) ); pAnycast = pAnycast->Next; } printf("\tNumber of Anycast Addresses: %d\n", i); @@ -606,8 +597,8 @@ ssdpDiscoverDevices(const char * const deviceTypes[], pMulticast = pCurrAddresses->FirstMulticastAddress; if (pMulticast) { for (i = 0; pMulticast != NULL; i++) { - IPAddr.S_un.S_addr = (u_long) pMultiCast->Address; - printf("\tMulticast Address[%d]: \t%s\n", i, inet_ntoa(IPAddr) ); + printf("\tMulticast Address[%d]: \t%s\n", i, inet_ntoa(((PSOCKADDR_IN)pMulticast->Address.lpSockaddr)->sin_addr) ); + pMulticast = pMulticast->Next; } } printf("\n"); @@ -647,7 +638,7 @@ ssdpDiscoverDevices(const char * const deviceTypes[], if(error) *error = MINISSDPC_SOCKET_ERROR; PRINT_SOCKET_ERROR("setsockopt(SO_REUSEADDR,...)"); - return NULL; + goto error; } if(ipv6) { @@ -692,7 +683,11 @@ ssdpDiscoverDevices(const char * const deviceTypes[], #endif } else { struct in_addr mc_if; +#if defined(_WIN32) && (_WIN32_WINNT >= _WIN32_WINNT_VISTA) + InetPtonA(AF_INET, multicastif, &mc_if); +#else mc_if.s_addr = inet_addr(multicastif); /* ex: 192.168.x.x */ +#endif if(mc_if.s_addr != INADDR_NONE) { ((struct sockaddr_in *)&sockudp_r)->sin_addr.s_addr = mc_if.s_addr; @@ -819,24 +814,26 @@ ssdpDiscoverDevices(const char * const deviceTypes[], fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); #endif break; - } - for(p = servinfo; p; p = p->ai_next) { - n = sendto(sudp, bufr, n, 0, p->ai_addr, p->ai_addrlen); - if (n < 0) { + } else { + struct addrinfo *p; + for(p = servinfo; p; p = p->ai_next) { + n = sendto(sudp, bufr, n, 0, p->ai_addr, MSC_CAST_INT p->ai_addrlen); + if (n < 0) { #ifdef DEBUG - char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; - if (getnameinfo(p->ai_addr, p->ai_addrlen, hbuf, sizeof(hbuf), sbuf, - sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0) { - fprintf(stderr, "host:%s port:%s\n", hbuf, sbuf); - } + char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; + if (getnameinfo(p->ai_addr, (socklen_t)p->ai_addrlen, hbuf, sizeof(hbuf), sbuf, + sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0) { + fprintf(stderr, "host:%s port:%s\n", hbuf, sbuf); + } #endif - PRINT_SOCKET_ERROR("sendto"); - continue; - } else { - sentok = 1; + PRINT_SOCKET_ERROR("sendto"); + continue; + } else { + sentok = 1; + } } + freeaddrinfo(servinfo); } - freeaddrinfo(servinfo); if(!sentok) { if(error) *error = MINISSDPC_SOCKET_ERROR; @@ -877,11 +874,11 @@ ssdpDiscoverDevices(const char * const deviceTypes[], stsize, st, usnsize, (usn?usn:""), urlsize, descURL); #endif /* DEBUG */ for(tmp=devlist; tmp; tmp = tmp->pNext) { - if(memcmp(tmp->descURL, descURL, urlsize) == 0 && + if(strncmp(tmp->descURL, descURL, urlsize) == 0 && tmp->descURL[urlsize] == '\0' && - memcmp(tmp->st, st, stsize) == 0 && + strncmp(tmp->st, st, stsize) == 0 && tmp->st[stsize] == '\0' && - (usnsize == 0 || memcmp(tmp->usn, usn, usnsize) == 0) && + (usnsize == 0 || strncmp(tmp->usn, usn, usnsize) == 0) && tmp->usn[usnsize] == '\0') break; } diff --git a/thirdparty/miniupnpc/minissdpc.h b/thirdparty/miniupnpc/miniupnpc/minissdpc.h similarity index 91% rename from thirdparty/miniupnpc/minissdpc.h rename to thirdparty/miniupnpc/miniupnpc/minissdpc.h index 167d897cb60..c99f929b9ea 100644 --- a/thirdparty/miniupnpc/minissdpc.h +++ b/thirdparty/miniupnpc/miniupnpc/minissdpc.h @@ -32,13 +32,13 @@ MINIUPNP_LIBSPEC int connectToMiniSSDPD(const char * socketpath); MINIUPNP_LIBSPEC int -disconnectFromMiniSSDPD(int fd); +disconnectFromMiniSSDPD(int s); MINIUPNP_LIBSPEC int -requestDevicesFromMiniSSDPD(int fd, const char * devtype); +requestDevicesFromMiniSSDPD(int s, const char * devtype); MINIUPNP_LIBSPEC struct UPNPDev * -receiveDevicesFromMiniSSDPD(int fd, int * error); +receiveDevicesFromMiniSSDPD(int s, int * error); #endif /* !(defined(_WIN32) || defined(__amigaos__) || defined(__amigaos4__)) */ diff --git a/thirdparty/miniupnpc/miniupnpc.c b/thirdparty/miniupnpc/miniupnpc/miniupnpc.c similarity index 97% rename from thirdparty/miniupnpc/miniupnpc.c rename to thirdparty/miniupnpc/miniupnpc/miniupnpc.c index 5d93ef99331..3181d10eb63 100644 --- a/thirdparty/miniupnpc/miniupnpc.c +++ b/thirdparty/miniupnpc/miniupnpc/miniupnpc.c @@ -1,9 +1,9 @@ -/* $Id: miniupnpc.c,v 1.149 2016/02/09 09:50:46 nanard Exp $ */ +/* $Id: miniupnpc.c,v 1.154 2019/04/23 12:12:13 nanard Exp $ */ /* vim: tabstop=4 shiftwidth=4 noexpandtab * Project : miniupnp * Web : http://miniupnp.free.fr/ * Author : Thomas BERNARD - * copyright (c) 2005-2018 Thomas Bernard + * copyright (c) 2005-2019 Thomas Bernard * This software is subjet to the conditions detailed in the * provided LICENSE file. */ #include @@ -63,7 +63,7 @@ #include "connecthostport.h" /* compare the beginning of a string with a constant string */ -#define COMPARE(str, cstr) (0==memcmp(str, cstr, sizeof(cstr) - 1)) +#define COMPARE(str, cstr) (0==strncmp(str, cstr, sizeof(cstr) - 1)) #ifndef MAXHOSTNAMELEN #define MAXHOSTNAMELEN 64 @@ -85,8 +85,7 @@ static int is_rfc1918addr(const char * addr) return 1; /* 172.16.0.0 - 172.31.255.255 (172.16/12 prefix) */ if(COMPARE(addr, "172.")) { - int i = atoi(addr + 4); - if((16 <= i) && (i <= 31)) + if((atoi(addr + 4) | 0x0f) == 0x1f) return 1; } return 0; @@ -416,7 +415,7 @@ static char * build_absolute_url(const char * baseurl, const char * descURL, const char * url, unsigned int scope_id) { - int l, n; + size_t l, n; char * s; const char * base; char * p; @@ -459,7 +458,7 @@ build_absolute_url(const char * baseurl, const char * descURL, memcpy(s, base, n); if(scope_id != 0) { s[n] = '\0'; - if(0 == memcmp(s, "http://[fe80:", 13)) { + if(n > 13 && 0 == memcmp(s, "http://[fe80:", 13)) { /* this is a linklocal IPv6 address */ p = strchr(s, ']'); if(p) { @@ -573,7 +572,6 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, int ndev = 0; int i; int state = -1; /* state 1 : IGD connected. State 2 : IGD. State 3 : anything */ - int n_igd = 0; char extIpAddr[16]; char myLanAddr[40]; int status_code = -1; @@ -588,12 +586,10 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, /* counting total number of devices in the list */ for(dev = devlist; dev; dev = dev->pNext) ndev++; - if(ndev > 0) - { - desc = calloc(ndev, sizeof(struct xml_desc)); - if(!desc) - return -1; /* memory allocation error */ - } + /* ndev is always > 0 */ + desc = calloc(ndev, sizeof(struct xml_desc)); + if(!desc) + return -1; /* memory allocation error */ /* Step 1 : downloading descriptions and testing type */ for(dev = devlist, i = 0; dev; dev = dev->pNext, i++) { @@ -617,7 +613,6 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) { desc[i].is_igd = 1; - n_igd++; if(lanaddr) strncpy(lanaddr, myLanAddr, lanaddrlen); } @@ -685,14 +680,9 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, } state = 0; free_and_return: - if(desc) { - for(i = 0; i < ndev; i++) { - if(desc[i].xml) { - free(desc[i].xml); - } - } - free(desc); - } + for(i = 0; i < ndev; i++) + free(desc[i].xml); + free(desc); return state; } @@ -717,7 +707,6 @@ UPNP_GetIGDFromUrl(const char * rootdescurl, memset(urls, 0, sizeof(struct UPNPUrls)); parserootdesc(descXML, descXMLsize, data); free(descXML); - descXML = NULL; GetUPNPUrls(urls, data, rootdescurl, 0); return 1; } else { diff --git a/thirdparty/miniupnpc/miniupnpc_declspec.h b/thirdparty/miniupnpc/miniupnpc/miniupnpc_declspec.h similarity index 100% rename from thirdparty/miniupnpc/miniupnpc_declspec.h rename to thirdparty/miniupnpc/miniupnpc/miniupnpc_declspec.h diff --git a/thirdparty/miniupnpc/miniupnpc_socketdef.h b/thirdparty/miniupnpc/miniupnpc/miniupnpc_socketdef.h similarity index 88% rename from thirdparty/miniupnpc/miniupnpc_socketdef.h rename to thirdparty/miniupnpc/miniupnpc/miniupnpc_socketdef.h index 965d9151b9e..d4f79a7bd61 100644 --- a/thirdparty/miniupnpc/miniupnpc_socketdef.h +++ b/thirdparty/miniupnpc/miniupnpc/miniupnpc_socketdef.h @@ -28,6 +28,13 @@ #endif +#ifdef _MSC_VER +#define MSC_CAST_INT (int) +#else +#define MSC_CAST_INT +#endif + +/* definition of PRINT_SOCKET_ERROR */ #ifdef _WIN32 #define PRINT_SOCKET_ERROR(x) fprintf(stderr, "Socket error: %s, %d\n", x, WSAGetLastError()); #else diff --git a/thirdparty/miniupnpc/miniupnpcmodule.c b/thirdparty/miniupnpc/miniupnpc/miniupnpcmodule.c similarity index 95% rename from thirdparty/miniupnpc/miniupnpcmodule.c rename to thirdparty/miniupnpc/miniupnpc/miniupnpcmodule.c index 8657a0e0027..d9341ab5bf3 100644 --- a/thirdparty/miniupnpc/miniupnpcmodule.c +++ b/thirdparty/miniupnpc/miniupnpc/miniupnpcmodule.c @@ -1,8 +1,9 @@ -/* $Id: miniupnpcmodule.c,v 1.24 2014/06/10 09:48:11 nanard Exp $*/ -/* Project : miniupnp +/* $Id: miniupnpcmodule.c,v 1.34 2019/05/20 19:07:16 nanard Exp $*/ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp * Author : Thomas BERNARD * website : https://miniupnp.tuxfamily.org/ - * copyright (c) 2007-2018 Thomas Bernard + * copyright (c) 2007-2019 Thomas Bernard * This software is subjet to the conditions detailed in the * provided LICENCE file. */ #include @@ -292,7 +293,7 @@ Py_END_ALLOW_THREADS } /* AddPortMapping(externalPort, protocol, internalHost, internalPort, desc, - * remoteHost) + * remoteHost, leaseDuration) * protocol is 'UDP' or 'TCP' */ static PyObject * UPnP_addportmapping(UPnPObject *self, PyObject *args) @@ -305,17 +306,24 @@ UPnP_addportmapping(UPnPObject *self, PyObject *args) const char * host; const char * desc; const char * remoteHost; - const char * leaseDuration = "0"; + unsigned int intLeaseDuration = 0; + char strLeaseDuration[12]; int r; - if (!PyArg_ParseTuple(args, "HssHzz", &ePort, &proto, - &host, &iPort, &desc, &remoteHost)) +#if (PY_MAJOR_VERSION >= 3) || (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION > 3) + if (!PyArg_ParseTuple(args, "HssHzz|I", &ePort, &proto, + &host, &iPort, &desc, &remoteHost, &intLeaseDuration)) +#else + if (!PyArg_ParseTuple(args, "HssHzz|i", &ePort, &proto, + &host, &iPort, &desc, &remoteHost, (int *)&intLeaseDuration)) +#endif return NULL; Py_BEGIN_ALLOW_THREADS sprintf(extPort, "%hu", ePort); sprintf(inPort, "%hu", iPort); + sprintf(strLeaseDuration, "%u", intLeaseDuration); r = UPNP_AddPortMapping(self->urls.controlURL, self->data.first.servicetype, extPort, inPort, host, desc, proto, - remoteHost, leaseDuration); + remoteHost, strLeaseDuration); Py_END_ALLOW_THREADS if(r==UPNPCOMMAND_SUCCESS) { @@ -676,6 +684,16 @@ initminiupnpc(void) /* initialize Winsock. */ WSADATA wsaData; int nResult = WSAStartup(MAKEWORD(2,2), &wsaData); + if (nResult != 0) + { + /* error code could be WSASYSNOTREADY WSASYSNOTREADY + * WSASYSNOTREADY WSASYSNOTREADY WSASYSNOTREADY */ +#if PY_MAJOR_VERSION >= 3 + return 0; +#else + return; +#endif + } UPnPType.tp_new = PyType_GenericNew; #endif diff --git a/thirdparty/miniupnpc/miniupnpcstrings.h b/thirdparty/miniupnpc/miniupnpc/miniupnpcstrings.h similarity index 100% rename from thirdparty/miniupnpc/miniupnpcstrings.h rename to thirdparty/miniupnpc/miniupnpc/miniupnpcstrings.h diff --git a/thirdparty/miniupnpc/miniupnpctypes.h b/thirdparty/miniupnpc/miniupnpc/miniupnpctypes.h similarity index 100% rename from thirdparty/miniupnpc/miniupnpctypes.h rename to thirdparty/miniupnpc/miniupnpc/miniupnpctypes.h diff --git a/thirdparty/miniupnpc/miniwget.c b/thirdparty/miniupnpc/miniupnpc/miniwget.c similarity index 99% rename from thirdparty/miniupnpc/miniwget.c rename to thirdparty/miniupnpc/miniupnpc/miniwget.c index a46ba760224..5c135f4efd5 100644 --- a/thirdparty/miniupnpc/miniwget.c +++ b/thirdparty/miniupnpc/miniupnpc/miniwget.c @@ -243,7 +243,7 @@ getHTTPResponse(SOCKET s, int * size, int * status_code) /* reading chunk size */ if(chunksize_buf_index == 0) { /* skipping any leading CR LF */ - if(icurelt, "NewPortListing") == 0) { /* specific case for NewPortListing which is a XML Document */ + free(data->portListing); data->portListing = malloc(l + 1); if(!data->portListing) { diff --git a/thirdparty/miniupnpc/upnpreplyparse.h b/thirdparty/miniupnpc/miniupnpc/upnpreplyparse.h similarity index 100% rename from thirdparty/miniupnpc/upnpreplyparse.h rename to thirdparty/miniupnpc/miniupnpc/upnpreplyparse.h diff --git a/thirdparty/miniupnpc/miniwget.h b/thirdparty/miniupnpc/miniwget.h deleted file mode 100644 index f5572c25449..00000000000 --- a/thirdparty/miniupnpc/miniwget.h +++ /dev/null @@ -1,27 +0,0 @@ -/* $Id: miniwget.h,v 1.12 2016/01/24 17:24:36 nanard Exp $ */ -/* Project : miniupnp - * Author : Thomas Bernard - * Copyright (c) 2005-2016 Thomas Bernard - * This software is subject to the conditions detailed in the - * LICENCE file provided in this distribution. - * */ -#ifndef MINIWGET_H_INCLUDED -#define MINIWGET_H_INCLUDED - -#include "miniupnpc_declspec.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MINIUPNP_LIBSPEC void * miniwget(const char *, int *, unsigned int, int *); - -MINIUPNP_LIBSPEC void * miniwget_getaddr(const char *, int *, char *, int, unsigned int, int *); - -int parseURL(const char *, char *, unsigned short *, char * *, unsigned int *); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/thirdparty/miniupnpc/upnpcommands.h b/thirdparty/miniupnpc/upnpcommands.h deleted file mode 100644 index 0c6d501666a..00000000000 --- a/thirdparty/miniupnpc/upnpcommands.h +++ /dev/null @@ -1,348 +0,0 @@ -/* $Id: upnpcommands.h,v 1.32 2018/03/13 23:34:47 nanard Exp $ */ -/* Miniupnp project : http://miniupnp.free.fr/ - * Author : Thomas Bernard - * Copyright (c) 2005-2018 Thomas Bernard - * This software is subject to the conditions detailed in the - * LICENCE file provided within this distribution */ -#ifndef UPNPCOMMANDS_H_INCLUDED -#define UPNPCOMMANDS_H_INCLUDED - -#include "miniupnpc_declspec.h" -#include "miniupnpctypes.h" - -/* MiniUPnPc return codes : */ -#define UPNPCOMMAND_SUCCESS (0) -#define UPNPCOMMAND_UNKNOWN_ERROR (-1) -#define UPNPCOMMAND_INVALID_ARGS (-2) -#define UPNPCOMMAND_HTTP_ERROR (-3) -#define UPNPCOMMAND_INVALID_RESPONSE (-4) -#define UPNPCOMMAND_MEM_ALLOC_ERROR (-5) - -#ifdef __cplusplus -extern "C" { -#endif - -struct PortMappingParserData; - -MINIUPNP_LIBSPEC UNSIGNED_INTEGER -UPNP_GetTotalBytesSent(const char * controlURL, - const char * servicetype); - -MINIUPNP_LIBSPEC UNSIGNED_INTEGER -UPNP_GetTotalBytesReceived(const char * controlURL, - const char * servicetype); - -MINIUPNP_LIBSPEC UNSIGNED_INTEGER -UPNP_GetTotalPacketsSent(const char * controlURL, - const char * servicetype); - -MINIUPNP_LIBSPEC UNSIGNED_INTEGER -UPNP_GetTotalPacketsReceived(const char * controlURL, - const char * servicetype); - -/* UPNP_GetStatusInfo() - * status and lastconnerror are 64 byte buffers - * Return values : - * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR - * or a UPnP Error code */ -MINIUPNP_LIBSPEC int -UPNP_GetStatusInfo(const char * controlURL, - const char * servicetype, - char * status, - unsigned int * uptime, - char * lastconnerror); - -/* UPNP_GetConnectionTypeInfo() - * argument connectionType is a 64 character buffer - * Return Values : - * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR - * or a UPnP Error code */ -MINIUPNP_LIBSPEC int -UPNP_GetConnectionTypeInfo(const char * controlURL, - const char * servicetype, - char * connectionType); - -/* UPNP_GetExternalIPAddress() call the corresponding UPNP method. - * if the third arg is not null the value is copied to it. - * at least 16 bytes must be available - * - * Return values : - * 0 : SUCCESS - * NON ZERO : ERROR Either an UPnP error code or an unknown error. - * - * possible UPnP Errors : - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 501 Action Failed - See UPnP Device Architecture section on Control. */ -MINIUPNP_LIBSPEC int -UPNP_GetExternalIPAddress(const char * controlURL, - const char * servicetype, - char * extIpAdd); - -/* UPNP_GetLinkLayerMaxBitRates() - * call WANCommonInterfaceConfig:1#GetCommonLinkProperties - * - * return values : - * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR - * or a UPnP Error Code. */ -MINIUPNP_LIBSPEC int -UPNP_GetLinkLayerMaxBitRates(const char* controlURL, - const char* servicetype, - unsigned int * bitrateDown, - unsigned int * bitrateUp); - -/* UPNP_AddPortMapping() - * if desc is NULL, it will be defaulted to "libminiupnpc" - * remoteHost is usually NULL because IGD don't support it. - * - * Return values : - * 0 : SUCCESS - * NON ZERO : ERROR. Either an UPnP error code or an unknown error. - * - * List of possible UPnP errors for AddPortMapping : - * errorCode errorDescription (short) - Description (long) - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 501 Action Failed - See UPnP Device Architecture section on Control. - * 606 Action not authorized - The action requested REQUIRES authorization and - * the sender was not authorized. - * 715 WildCardNotPermittedInSrcIP - The source IP address cannot be - * wild-carded - * 716 WildCardNotPermittedInExtPort - The external port cannot be wild-carded - * 718 ConflictInMappingEntry - The port mapping entry specified conflicts - * with a mapping assigned previously to another client - * 724 SamePortValuesRequired - Internal and External port values - * must be the same - * 725 OnlyPermanentLeasesSupported - The NAT implementation only supports - * permanent lease times on port mappings - * 726 RemoteHostOnlySupportsWildcard - RemoteHost must be a wildcard - * and cannot be a specific IP address or DNS name - * 727 ExternalPortOnlySupportsWildcard - ExternalPort must be a wildcard and - * cannot be a specific port value - * 728 NoPortMapsAvailable - There are not enough free ports available to - * complete port mapping. - * 729 ConflictWithOtherMechanisms - Attempted port mapping is not allowed - * due to conflict with other mechanisms. - * 732 WildCardNotPermittedInIntPort - The internal port cannot be wild-carded - */ -MINIUPNP_LIBSPEC int -UPNP_AddPortMapping(const char * controlURL, const char * servicetype, - const char * extPort, - const char * inPort, - const char * inClient, - const char * desc, - const char * proto, - const char * remoteHost, - const char * leaseDuration); - -/* UPNP_AddAnyPortMapping() - * if desc is NULL, it will be defaulted to "libminiupnpc" - * remoteHost is usually NULL because IGD don't support it. - * - * Return values : - * 0 : SUCCESS - * NON ZERO : ERROR. Either an UPnP error code or an unknown error. - * - * List of possible UPnP errors for AddPortMapping : - * errorCode errorDescription (short) - Description (long) - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 501 Action Failed - See UPnP Device Architecture section on Control. - * 606 Action not authorized - The action requested REQUIRES authorization and - * the sender was not authorized. - * 715 WildCardNotPermittedInSrcIP - The source IP address cannot be - * wild-carded - * 716 WildCardNotPermittedInExtPort - The external port cannot be wild-carded - * 728 NoPortMapsAvailable - There are not enough free ports available to - * complete port mapping. - * 729 ConflictWithOtherMechanisms - Attempted port mapping is not allowed - * due to conflict with other mechanisms. - * 732 WildCardNotPermittedInIntPort - The internal port cannot be wild-carded - */ -MINIUPNP_LIBSPEC int -UPNP_AddAnyPortMapping(const char * controlURL, const char * servicetype, - const char * extPort, - const char * inPort, - const char * inClient, - const char * desc, - const char * proto, - const char * remoteHost, - const char * leaseDuration, - char * reservedPort); - -/* UPNP_DeletePortMapping() - * Use same argument values as what was used for AddPortMapping(). - * remoteHost is usually NULL because IGD don't support it. - * Return Values : - * 0 : SUCCESS - * NON ZERO : error. Either an UPnP error code or an undefined error. - * - * List of possible UPnP errors for DeletePortMapping : - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 606 Action not authorized - The action requested REQUIRES authorization - * and the sender was not authorized. - * 714 NoSuchEntryInArray - The specified value does not exist in the array */ -MINIUPNP_LIBSPEC int -UPNP_DeletePortMapping(const char * controlURL, const char * servicetype, - const char * extPort, const char * proto, - const char * remoteHost); - -/* UPNP_DeletePortRangeMapping() - * Use same argument values as what was used for AddPortMapping(). - * remoteHost is usually NULL because IGD don't support it. - * Return Values : - * 0 : SUCCESS - * NON ZERO : error. Either an UPnP error code or an undefined error. - * - * List of possible UPnP errors for DeletePortMapping : - * 606 Action not authorized - The action requested REQUIRES authorization - * and the sender was not authorized. - * 730 PortMappingNotFound - This error message is returned if no port - * mapping is found in the specified range. - * 733 InconsistentParameters - NewStartPort and NewEndPort values are not consistent. */ -MINIUPNP_LIBSPEC int -UPNP_DeletePortMappingRange(const char * controlURL, const char * servicetype, - const char * extPortStart, const char * extPortEnd, - const char * proto, - const char * manage); - -/* UPNP_GetPortMappingNumberOfEntries() - * not supported by all routers */ -MINIUPNP_LIBSPEC int -UPNP_GetPortMappingNumberOfEntries(const char* controlURL, - const char* servicetype, - unsigned int * num); - -/* UPNP_GetSpecificPortMappingEntry() - * retrieves an existing port mapping - * params : - * in extPort - * in proto - * in remoteHost - * out intClient (16 bytes) - * out intPort (6 bytes) - * out desc (80 bytes) - * out enabled (4 bytes) - * out leaseDuration (16 bytes) - * - * return value : - * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR - * or a UPnP Error Code. - * - * List of possible UPnP errors for _GetSpecificPortMappingEntry : - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 501 Action Failed - See UPnP Device Architecture section on Control. - * 606 Action not authorized - The action requested REQUIRES authorization - * and the sender was not authorized. - * 714 NoSuchEntryInArray - The specified value does not exist in the array. - */ -MINIUPNP_LIBSPEC int -UPNP_GetSpecificPortMappingEntry(const char * controlURL, - const char * servicetype, - const char * extPort, - const char * proto, - const char * remoteHost, - char * intClient, - char * intPort, - char * desc, - char * enabled, - char * leaseDuration); - -/* UPNP_GetGenericPortMappingEntry() - * params : - * in index - * out extPort (6 bytes) - * out intClient (16 bytes) - * out intPort (6 bytes) - * out protocol (4 bytes) - * out desc (80 bytes) - * out enabled (4 bytes) - * out rHost (64 bytes) - * out duration (16 bytes) - * - * return value : - * UPNPCOMMAND_SUCCESS, UPNPCOMMAND_INVALID_ARGS, UPNPCOMMAND_UNKNOWN_ERROR - * or a UPnP Error Code. - * - * Possible UPNP Error codes : - * 402 Invalid Args - See UPnP Device Architecture section on Control. - * 606 Action not authorized - The action requested REQUIRES authorization - * and the sender was not authorized. - * 713 SpecifiedArrayIndexInvalid - The specified array index is out of bounds - */ -MINIUPNP_LIBSPEC int -UPNP_GetGenericPortMappingEntry(const char * controlURL, - const char * servicetype, - const char * index, - char * extPort, - char * intClient, - char * intPort, - char * protocol, - char * desc, - char * enabled, - char * rHost, - char * duration); - -/* UPNP_GetListOfPortMappings() Available in IGD v2 - * - * - * Possible UPNP Error codes : - * 606 Action not Authorized - * 730 PortMappingNotFound - no port mapping is found in the specified range. - * 733 InconsistantParameters - NewStartPort and NewEndPort values are not - * consistent. - */ -MINIUPNP_LIBSPEC int -UPNP_GetListOfPortMappings(const char * controlURL, - const char * servicetype, - const char * startPort, - const char * endPort, - const char * protocol, - const char * numberOfPorts, - struct PortMappingParserData * data); - -/* IGD:2, functions for service WANIPv6FirewallControl:1 */ -MINIUPNP_LIBSPEC int -UPNP_GetFirewallStatus(const char * controlURL, - const char * servicetype, - int * firewallEnabled, - int * inboundPinholeAllowed); - -MINIUPNP_LIBSPEC int -UPNP_GetOutboundPinholeTimeout(const char * controlURL, const char * servicetype, - const char * remoteHost, - const char * remotePort, - const char * intClient, - const char * intPort, - const char * proto, - int * opTimeout); - -MINIUPNP_LIBSPEC int -UPNP_AddPinhole(const char * controlURL, const char * servicetype, - const char * remoteHost, - const char * remotePort, - const char * intClient, - const char * intPort, - const char * proto, - const char * leaseTime, - char * uniqueID); - -MINIUPNP_LIBSPEC int -UPNP_UpdatePinhole(const char * controlURL, const char * servicetype, - const char * uniqueID, - const char * leaseTime); - -MINIUPNP_LIBSPEC int -UPNP_DeletePinhole(const char * controlURL, const char * servicetype, const char * uniqueID); - -MINIUPNP_LIBSPEC int -UPNP_CheckPinholeWorking(const char * controlURL, const char * servicetype, - const char * uniqueID, int * isWorking); - -MINIUPNP_LIBSPEC int -UPNP_GetPinholePackets(const char * controlURL, const char * servicetype, - const char * uniqueID, int * packets); - -#ifdef __cplusplus -} -#endif - -#endif - From cfa50d67d2e697cdff6853faa40c88f18bfbf485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Wed, 10 Jul 2019 16:15:01 +0200 Subject: [PATCH 003/113] mbedtls: Update to upstream version 2.16.2 (cherry picked from commit 6321cc8da3fb6dd5fd9ce3e6a1c5496c1c0d65ae) --- thirdparty/README.md | 4 +- thirdparty/mbedtls/include/mbedtls/aes.h | 4 + thirdparty/mbedtls/include/mbedtls/aesni.h | 6 + thirdparty/mbedtls/include/mbedtls/arc4.h | 4 + .../mbedtls/include/mbedtls/asn1write.h | 13 +- thirdparty/mbedtls/include/mbedtls/base64.h | 9 + thirdparty/mbedtls/include/mbedtls/bignum.h | 11 +- thirdparty/mbedtls/include/mbedtls/bn_mul.h | 8 +- thirdparty/mbedtls/include/mbedtls/camellia.h | 4 + thirdparty/mbedtls/include/mbedtls/ccm.h | 6 + thirdparty/mbedtls/include/mbedtls/certs.h | 246 +- thirdparty/mbedtls/include/mbedtls/cipher.h | 2 +- thirdparty/mbedtls/include/mbedtls/cmac.h | 6 + .../mbedtls/include/mbedtls/compat-1.3.h | 6 + thirdparty/mbedtls/include/mbedtls/config.h | 24 + thirdparty/mbedtls/include/mbedtls/ctr_drbg.h | 10 + thirdparty/mbedtls/include/mbedtls/des.h | 4 + thirdparty/mbedtls/include/mbedtls/dhm.h | 3 + thirdparty/mbedtls/include/mbedtls/ecdh.h | 6 + thirdparty/mbedtls/include/mbedtls/ecdsa.h | 6 + thirdparty/mbedtls/include/mbedtls/ecjpake.h | 5 + thirdparty/mbedtls/include/mbedtls/ecp.h | 132 +- .../mbedtls/include/mbedtls/ecp_internal.h | 6 + thirdparty/mbedtls/include/mbedtls/error.h | 6 + thirdparty/mbedtls/include/mbedtls/gcm.h | 10 + thirdparty/mbedtls/include/mbedtls/havege.h | 6 + thirdparty/mbedtls/include/mbedtls/hkdf.h | 6 + .../mbedtls/include/mbedtls/hmac_drbg.h | 6 + thirdparty/mbedtls/include/mbedtls/md2.h | 4 + thirdparty/mbedtls/include/mbedtls/md4.h | 4 + thirdparty/mbedtls/include/mbedtls/md5.h | 4 + thirdparty/mbedtls/include/mbedtls/net.h | 5 + thirdparty/mbedtls/include/mbedtls/nist_kw.h | 6 + thirdparty/mbedtls/include/mbedtls/padlock.h | 8 +- thirdparty/mbedtls/include/mbedtls/pem.h | 6 + thirdparty/mbedtls/include/mbedtls/pkcs12.h | 6 + thirdparty/mbedtls/include/mbedtls/pkcs5.h | 10 + .../mbedtls/include/mbedtls/platform_util.h | 4 +- thirdparty/mbedtls/include/mbedtls/poly1305.h | 2 +- .../mbedtls/include/mbedtls/ripemd160.h | 4 + thirdparty/mbedtls/include/mbedtls/rsa.h | 4 + thirdparty/mbedtls/include/mbedtls/sha1.h | 4 + thirdparty/mbedtls/include/mbedtls/sha256.h | 4 + thirdparty/mbedtls/include/mbedtls/sha512.h | 5 + thirdparty/mbedtls/include/mbedtls/ssl.h | 20 +- .../mbedtls/include/mbedtls/ssl_cache.h | 6 + .../include/mbedtls/ssl_ciphersuites.h | 6 + .../mbedtls/include/mbedtls/ssl_cookie.h | 6 + .../mbedtls/include/mbedtls/ssl_internal.h | 6 + .../mbedtls/include/mbedtls/ssl_ticket.h | 6 + thirdparty/mbedtls/include/mbedtls/version.h | 8 +- thirdparty/mbedtls/include/mbedtls/x509.h | 4 + thirdparty/mbedtls/include/mbedtls/x509_crt.h | 2 +- thirdparty/mbedtls/include/mbedtls/x509_csr.h | 8 + thirdparty/mbedtls/include/mbedtls/xtea.h | 4 + thirdparty/mbedtls/library/aes.c | 96 +- thirdparty/mbedtls/library/asn1write.c | 28 +- thirdparty/mbedtls/library/bignum.c | 191 +- thirdparty/mbedtls/library/ccm.c | 16 +- thirdparty/mbedtls/library/certs.c | 2024 ++++++++++++++--- thirdparty/mbedtls/library/chacha20.c | 10 +- thirdparty/mbedtls/library/debug.c | 44 +- thirdparty/mbedtls/library/des.c | 77 +- thirdparty/mbedtls/library/dhm.c | 16 + thirdparty/mbedtls/library/ecdh.c | 27 +- thirdparty/mbedtls/library/ecdsa.c | 9 +- thirdparty/mbedtls/library/ecp.c | 20 +- thirdparty/mbedtls/library/ecp_curves.c | 60 +- thirdparty/mbedtls/library/entropy_poll.c | 29 +- thirdparty/mbedtls/library/havege.c | 2 +- thirdparty/mbedtls/library/md4.c | 30 +- thirdparty/mbedtls/library/md5.c | 21 +- thirdparty/mbedtls/library/oid.c | 53 +- thirdparty/mbedtls/library/poly1305.c | 8 +- thirdparty/mbedtls/library/ripemd160.c | 33 +- thirdparty/mbedtls/library/sha1.c | 32 +- thirdparty/mbedtls/library/sha256.c | 29 +- thirdparty/mbedtls/library/sha512.c | 21 +- thirdparty/mbedtls/library/ssl_ciphersuites.c | 57 +- thirdparty/mbedtls/library/version_features.c | 3 + thirdparty/mbedtls/library/x509.c | 62 +- thirdparty/mbedtls/library/x509_crl.c | 10 +- thirdparty/mbedtls/library/x509_crt.c | 51 +- thirdparty/mbedtls/library/x509_csr.c | 15 +- thirdparty/mbedtls/library/x509write_crt.c | 45 +- thirdparty/mbedtls/library/x509write_csr.c | 36 +- 86 files changed, 2994 insertions(+), 876 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index 702943e1bc0..d1eaec03eef 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -285,10 +285,10 @@ Godot build configurations, check them out when updating. ## mbedtls - Upstream: https://tls.mbed.org/ -- Version: 2.16.0 +- Version: 2.16.2 - License: Apache 2.0 -File extracted from upstream release tarball `mbedtls-2.16.0-apache.tgz`: +File extracted from upstream release tarball (`-apache.tgz` variant): - All `*.h` from `include/mbedtls/` to `thirdparty/mbedtls/include/mbedtls/` - All `*.c` from `library/` to `thirdparty/mbedtls/library/` - Applied the patch in `thirdparty/mbedtls/1453.diff` (PR 1453). Soon to be merged upstream. Check it out at next update. diff --git a/thirdparty/mbedtls/include/mbedtls/aes.h b/thirdparty/mbedtls/include/mbedtls/aes.h index b42e564efcd..94e7282d36d 100644 --- a/thirdparty/mbedtls/include/mbedtls/aes.h +++ b/thirdparty/mbedtls/include/mbedtls/aes.h @@ -655,6 +655,8 @@ MBEDTLS_DEPRECATED void mbedtls_aes_decrypt( mbedtls_aes_context *ctx, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ + +#if defined(MBEDTLS_SELF_TEST) /** * \brief Checkup routine. * @@ -663,6 +665,8 @@ MBEDTLS_DEPRECATED void mbedtls_aes_decrypt( mbedtls_aes_context *ctx, */ int mbedtls_aes_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/aesni.h b/thirdparty/mbedtls/include/mbedtls/aesni.h index 0196f49b872..a4ca012f8a1 100644 --- a/thirdparty/mbedtls/include/mbedtls/aesni.h +++ b/thirdparty/mbedtls/include/mbedtls/aesni.h @@ -27,6 +27,12 @@ #ifndef MBEDTLS_AESNI_H #define MBEDTLS_AESNI_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "aes.h" #define MBEDTLS_AESNI_AES 0x02000000u diff --git a/thirdparty/mbedtls/include/mbedtls/arc4.h b/thirdparty/mbedtls/include/mbedtls/arc4.h index c43f4065f13..fb044d5b7fd 100644 --- a/thirdparty/mbedtls/include/mbedtls/arc4.h +++ b/thirdparty/mbedtls/include/mbedtls/arc4.h @@ -123,6 +123,8 @@ void mbedtls_arc4_setup( mbedtls_arc4_context *ctx, const unsigned char *key, int mbedtls_arc4_crypt( mbedtls_arc4_context *ctx, size_t length, const unsigned char *input, unsigned char *output ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -135,6 +137,8 @@ int mbedtls_arc4_crypt( mbedtls_arc4_context *ctx, size_t length, const unsigned */ int mbedtls_arc4_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/asn1write.h b/thirdparty/mbedtls/include/mbedtls/asn1write.h index 76c1780b595..a194243696a 100644 --- a/thirdparty/mbedtls/include/mbedtls/asn1write.h +++ b/thirdparty/mbedtls/include/mbedtls/asn1write.h @@ -24,14 +24,21 @@ #ifndef MBEDTLS_ASN1_WRITE_H #define MBEDTLS_ASN1_WRITE_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "asn1.h" #define MBEDTLS_ASN1_CHK_ADD(g, f) \ - do { \ - if( ( ret = f ) < 0 ) \ + do \ + { \ + if( ( ret = (f) ) < 0 ) \ return( ret ); \ else \ - g += ret; \ + (g) += ret; \ } while( 0 ) #ifdef __cplusplus diff --git a/thirdparty/mbedtls/include/mbedtls/base64.h b/thirdparty/mbedtls/include/mbedtls/base64.h index 7a64f521635..0d024164c56 100644 --- a/thirdparty/mbedtls/include/mbedtls/base64.h +++ b/thirdparty/mbedtls/include/mbedtls/base64.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_BASE64_H #define MBEDTLS_BASE64_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include #define MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL -0x002A /**< Output buffer too small. */ @@ -75,6 +81,7 @@ int mbedtls_base64_encode( unsigned char *dst, size_t dlen, size_t *olen, int mbedtls_base64_decode( unsigned char *dst, size_t dlen, size_t *olen, const unsigned char *src, size_t slen ); +#if defined(MBEDTLS_SELF_TEST) /** * \brief Checkup routine * @@ -82,6 +89,8 @@ int mbedtls_base64_decode( unsigned char *dst, size_t dlen, size_t *olen, */ int mbedtls_base64_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/bignum.h b/thirdparty/mbedtls/include/mbedtls/bignum.h index 141a8e9adf4..1c8607264fe 100644 --- a/thirdparty/mbedtls/include/mbedtls/bignum.h +++ b/thirdparty/mbedtls/include/mbedtls/bignum.h @@ -46,7 +46,12 @@ #define MBEDTLS_ERR_MPI_NOT_ACCEPTABLE -0x000E /**< The input arguments are not acceptable. */ #define MBEDTLS_ERR_MPI_ALLOC_FAILED -0x0010 /**< Memory allocation failed. */ -#define MBEDTLS_MPI_CHK(f) do { if( ( ret = f ) != 0 ) goto cleanup; } while( 0 ) +#define MBEDTLS_MPI_CHK(f) \ + do \ + { \ + if( ( ret = (f) ) != 0 ) \ + goto cleanup; \ + } while( 0 ) /* * Maximum size MPIs are allowed to grow to in number of limbs. @@ -943,6 +948,8 @@ int mbedtls_mpi_gen_prime( mbedtls_mpi *X, size_t nbits, int flags, int (*f_rng)(void *, unsigned char *, size_t), void *p_rng ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -950,6 +957,8 @@ int mbedtls_mpi_gen_prime( mbedtls_mpi *X, size_t nbits, int flags, */ int mbedtls_mpi_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/bn_mul.h b/thirdparty/mbedtls/include/mbedtls/bn_mul.h index 2f7b72fe4cd..c33bd8d4ab5 100644 --- a/thirdparty/mbedtls/include/mbedtls/bn_mul.h +++ b/thirdparty/mbedtls/include/mbedtls/bn_mul.h @@ -38,6 +38,12 @@ #ifndef MBEDTLS_BN_MUL_H #define MBEDTLS_BN_MUL_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "bignum.h" #if defined(MBEDTLS_HAVE_ASM) @@ -750,7 +756,7 @@ "sw $10, %2 \n\t" \ : "=m" (c), "=m" (d), "=m" (s) \ : "m" (s), "m" (d), "m" (c), "m" (b) \ - : "$9", "$10", "$11", "$12", "$13", "$14", "$15" \ + : "$9", "$10", "$11", "$12", "$13", "$14", "$15", "lo", "hi" \ ); #endif /* MIPS */ diff --git a/thirdparty/mbedtls/include/mbedtls/camellia.h b/thirdparty/mbedtls/include/mbedtls/camellia.h index 0f7c42c92d5..3eeb66366d6 100644 --- a/thirdparty/mbedtls/include/mbedtls/camellia.h +++ b/thirdparty/mbedtls/include/mbedtls/camellia.h @@ -308,6 +308,8 @@ int mbedtls_camellia_crypt_ctr( mbedtls_camellia_context *ctx, unsigned char *output ); #endif /* MBEDTLS_CIPHER_MODE_CTR */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -315,6 +317,8 @@ int mbedtls_camellia_crypt_ctr( mbedtls_camellia_context *ctx, */ int mbedtls_camellia_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/ccm.h b/thirdparty/mbedtls/include/mbedtls/ccm.h index 3f6b8f67095..f03e3b580ea 100644 --- a/thirdparty/mbedtls/include/mbedtls/ccm.h +++ b/thirdparty/mbedtls/include/mbedtls/ccm.h @@ -49,6 +49,12 @@ #ifndef MBEDTLS_CCM_H #define MBEDTLS_CCM_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "cipher.h" #define MBEDTLS_ERR_CCM_BAD_INPUT -0x000D /**< Bad input parameters to the function. */ diff --git a/thirdparty/mbedtls/include/mbedtls/certs.h b/thirdparty/mbedtls/include/mbedtls/certs.h index 8dab7b5ce8c..179ebbbad22 100644 --- a/thirdparty/mbedtls/include/mbedtls/certs.h +++ b/thirdparty/mbedtls/include/mbedtls/certs.h @@ -24,74 +24,226 @@ #ifndef MBEDTLS_CERTS_H #define MBEDTLS_CERTS_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include #ifdef __cplusplus extern "C" { #endif +/* List of all PEM-encoded CA certificates, terminated by NULL; + * PEM encoded if MBEDTLS_PEM_PARSE_C is enabled, DER encoded + * otherwise. */ +extern const char * mbedtls_test_cas[]; +extern const size_t mbedtls_test_cas_len[]; + +/* List of all DER-encoded CA certificates, terminated by NULL */ +extern const unsigned char * mbedtls_test_cas_der[]; +extern const size_t mbedtls_test_cas_der_len[]; + #if defined(MBEDTLS_PEM_PARSE_C) /* Concatenation of all CA certificates in PEM format if available */ extern const char mbedtls_test_cas_pem[]; extern const size_t mbedtls_test_cas_pem_len; -#endif - -/* List of all CA certificates, terminated by NULL */ -extern const char * mbedtls_test_cas[]; -extern const size_t mbedtls_test_cas_len[]; +#endif /* MBEDTLS_PEM_PARSE_C */ /* - * Convenience for users who just want a certificate: - * RSA by default, or ECDSA if RSA is not available + * CA test certificates */ -extern const char * mbedtls_test_ca_crt; -extern const size_t mbedtls_test_ca_crt_len; -extern const char * mbedtls_test_ca_key; -extern const size_t mbedtls_test_ca_key_len; -extern const char * mbedtls_test_ca_pwd; -extern const size_t mbedtls_test_ca_pwd_len; -extern const char * mbedtls_test_srv_crt; -extern const size_t mbedtls_test_srv_crt_len; -extern const char * mbedtls_test_srv_key; -extern const size_t mbedtls_test_srv_key_len; -extern const char * mbedtls_test_cli_crt; -extern const size_t mbedtls_test_cli_crt_len; -extern const char * mbedtls_test_cli_key; -extern const size_t mbedtls_test_cli_key_len; -#if defined(MBEDTLS_ECDSA_C) -extern const char mbedtls_test_ca_crt_ec[]; +extern const char mbedtls_test_ca_crt_ec_pem[]; +extern const char mbedtls_test_ca_key_ec_pem[]; +extern const char mbedtls_test_ca_pwd_ec_pem[]; +extern const char mbedtls_test_ca_key_rsa_pem[]; +extern const char mbedtls_test_ca_pwd_rsa_pem[]; +extern const char mbedtls_test_ca_crt_rsa_sha1_pem[]; +extern const char mbedtls_test_ca_crt_rsa_sha256_pem[]; + +extern const unsigned char mbedtls_test_ca_crt_ec_der[]; +extern const unsigned char mbedtls_test_ca_key_ec_der[]; +extern const unsigned char mbedtls_test_ca_key_rsa_der[]; +extern const unsigned char mbedtls_test_ca_crt_rsa_sha1_der[]; +extern const unsigned char mbedtls_test_ca_crt_rsa_sha256_der[]; + +extern const size_t mbedtls_test_ca_crt_ec_pem_len; +extern const size_t mbedtls_test_ca_key_ec_pem_len; +extern const size_t mbedtls_test_ca_pwd_ec_pem_len; +extern const size_t mbedtls_test_ca_key_rsa_pem_len; +extern const size_t mbedtls_test_ca_pwd_rsa_pem_len; +extern const size_t mbedtls_test_ca_crt_rsa_sha1_pem_len; +extern const size_t mbedtls_test_ca_crt_rsa_sha256_pem_len; + +extern const size_t mbedtls_test_ca_crt_ec_der_len; +extern const size_t mbedtls_test_ca_key_ec_der_len; +extern const size_t mbedtls_test_ca_pwd_ec_der_len; +extern const size_t mbedtls_test_ca_key_rsa_der_len; +extern const size_t mbedtls_test_ca_pwd_rsa_der_len; +extern const size_t mbedtls_test_ca_crt_rsa_sha1_der_len; +extern const size_t mbedtls_test_ca_crt_rsa_sha256_der_len; + +/* Config-dependent dispatch between PEM and DER encoding + * (PEM if enabled, otherwise DER) */ + +extern const char mbedtls_test_ca_crt_ec[]; +extern const char mbedtls_test_ca_key_ec[]; +extern const char mbedtls_test_ca_pwd_ec[]; +extern const char mbedtls_test_ca_key_rsa[]; +extern const char mbedtls_test_ca_pwd_rsa[]; +extern const char mbedtls_test_ca_crt_rsa_sha1[]; +extern const char mbedtls_test_ca_crt_rsa_sha256[]; + extern const size_t mbedtls_test_ca_crt_ec_len; -extern const char mbedtls_test_ca_key_ec[]; extern const size_t mbedtls_test_ca_key_ec_len; -extern const char mbedtls_test_ca_pwd_ec[]; extern const size_t mbedtls_test_ca_pwd_ec_len; -extern const char mbedtls_test_srv_crt_ec[]; -extern const size_t mbedtls_test_srv_crt_ec_len; -extern const char mbedtls_test_srv_key_ec[]; -extern const size_t mbedtls_test_srv_key_ec_len; -extern const char mbedtls_test_cli_crt_ec[]; -extern const size_t mbedtls_test_cli_crt_ec_len; -extern const char mbedtls_test_cli_key_ec[]; -extern const size_t mbedtls_test_cli_key_ec_len; -#endif - -#if defined(MBEDTLS_RSA_C) -extern const char mbedtls_test_ca_crt_rsa[]; -extern const size_t mbedtls_test_ca_crt_rsa_len; -extern const char mbedtls_test_ca_key_rsa[]; extern const size_t mbedtls_test_ca_key_rsa_len; -extern const char mbedtls_test_ca_pwd_rsa[]; extern const size_t mbedtls_test_ca_pwd_rsa_len; -extern const char mbedtls_test_srv_crt_rsa[]; -extern const size_t mbedtls_test_srv_crt_rsa_len; -extern const char mbedtls_test_srv_key_rsa[]; +extern const size_t mbedtls_test_ca_crt_rsa_sha1_len; +extern const size_t mbedtls_test_ca_crt_rsa_sha256_len; + +/* Config-dependent dispatch between SHA-1 and SHA-256 + * (SHA-256 if enabled, otherwise SHA-1) */ + +extern const char mbedtls_test_ca_crt_rsa[]; +extern const size_t mbedtls_test_ca_crt_rsa_len; + +/* Config-dependent dispatch between EC and RSA + * (RSA if enabled, otherwise EC) */ + +extern const char * mbedtls_test_ca_crt; +extern const char * mbedtls_test_ca_key; +extern const char * mbedtls_test_ca_pwd; +extern const size_t mbedtls_test_ca_crt_len; +extern const size_t mbedtls_test_ca_key_len; +extern const size_t mbedtls_test_ca_pwd_len; + +/* + * Server test certificates + */ + +extern const char mbedtls_test_srv_crt_ec_pem[]; +extern const char mbedtls_test_srv_key_ec_pem[]; +extern const char mbedtls_test_srv_pwd_ec_pem[]; +extern const char mbedtls_test_srv_key_rsa_pem[]; +extern const char mbedtls_test_srv_pwd_rsa_pem[]; +extern const char mbedtls_test_srv_crt_rsa_sha1_pem[]; +extern const char mbedtls_test_srv_crt_rsa_sha256_pem[]; + +extern const unsigned char mbedtls_test_srv_crt_ec_der[]; +extern const unsigned char mbedtls_test_srv_key_ec_der[]; +extern const unsigned char mbedtls_test_srv_key_rsa_der[]; +extern const unsigned char mbedtls_test_srv_crt_rsa_sha1_der[]; +extern const unsigned char mbedtls_test_srv_crt_rsa_sha256_der[]; + +extern const size_t mbedtls_test_srv_crt_ec_pem_len; +extern const size_t mbedtls_test_srv_key_ec_pem_len; +extern const size_t mbedtls_test_srv_pwd_ec_pem_len; +extern const size_t mbedtls_test_srv_key_rsa_pem_len; +extern const size_t mbedtls_test_srv_pwd_rsa_pem_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha1_pem_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha256_pem_len; + +extern const size_t mbedtls_test_srv_crt_ec_der_len; +extern const size_t mbedtls_test_srv_key_ec_der_len; +extern const size_t mbedtls_test_srv_pwd_ec_der_len; +extern const size_t mbedtls_test_srv_key_rsa_der_len; +extern const size_t mbedtls_test_srv_pwd_rsa_der_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha1_der_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha256_der_len; + +/* Config-dependent dispatch between PEM and DER encoding + * (PEM if enabled, otherwise DER) */ + +extern const char mbedtls_test_srv_crt_ec[]; +extern const char mbedtls_test_srv_key_ec[]; +extern const char mbedtls_test_srv_pwd_ec[]; +extern const char mbedtls_test_srv_key_rsa[]; +extern const char mbedtls_test_srv_pwd_rsa[]; +extern const char mbedtls_test_srv_crt_rsa_sha1[]; +extern const char mbedtls_test_srv_crt_rsa_sha256[]; + +extern const size_t mbedtls_test_srv_crt_ec_len; +extern const size_t mbedtls_test_srv_key_ec_len; +extern const size_t mbedtls_test_srv_pwd_ec_len; extern const size_t mbedtls_test_srv_key_rsa_len; -extern const char mbedtls_test_cli_crt_rsa[]; -extern const size_t mbedtls_test_cli_crt_rsa_len; -extern const char mbedtls_test_cli_key_rsa[]; +extern const size_t mbedtls_test_srv_pwd_rsa_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha1_len; +extern const size_t mbedtls_test_srv_crt_rsa_sha256_len; + +/* Config-dependent dispatch between SHA-1 and SHA-256 + * (SHA-256 if enabled, otherwise SHA-1) */ + +extern const char mbedtls_test_srv_crt_rsa[]; +extern const size_t mbedtls_test_srv_crt_rsa_len; + +/* Config-dependent dispatch between EC and RSA + * (RSA if enabled, otherwise EC) */ + +extern const char * mbedtls_test_srv_crt; +extern const char * mbedtls_test_srv_key; +extern const char * mbedtls_test_srv_pwd; +extern const size_t mbedtls_test_srv_crt_len; +extern const size_t mbedtls_test_srv_key_len; +extern const size_t mbedtls_test_srv_pwd_len; + +/* + * Client test certificates + */ + +extern const char mbedtls_test_cli_crt_ec_pem[]; +extern const char mbedtls_test_cli_key_ec_pem[]; +extern const char mbedtls_test_cli_pwd_ec_pem[]; +extern const char mbedtls_test_cli_key_rsa_pem[]; +extern const char mbedtls_test_cli_pwd_rsa_pem[]; +extern const char mbedtls_test_cli_crt_rsa_pem[]; + +extern const unsigned char mbedtls_test_cli_crt_ec_der[]; +extern const unsigned char mbedtls_test_cli_key_ec_der[]; +extern const unsigned char mbedtls_test_cli_key_rsa_der[]; +extern const unsigned char mbedtls_test_cli_crt_rsa_der[]; + +extern const size_t mbedtls_test_cli_crt_ec_pem_len; +extern const size_t mbedtls_test_cli_key_ec_pem_len; +extern const size_t mbedtls_test_cli_pwd_ec_pem_len; +extern const size_t mbedtls_test_cli_key_rsa_pem_len; +extern const size_t mbedtls_test_cli_pwd_rsa_pem_len; +extern const size_t mbedtls_test_cli_crt_rsa_pem_len; + +extern const size_t mbedtls_test_cli_crt_ec_der_len; +extern const size_t mbedtls_test_cli_key_ec_der_len; +extern const size_t mbedtls_test_cli_key_rsa_der_len; +extern const size_t mbedtls_test_cli_crt_rsa_der_len; + +/* Config-dependent dispatch between PEM and DER encoding + * (PEM if enabled, otherwise DER) */ + +extern const char mbedtls_test_cli_crt_ec[]; +extern const char mbedtls_test_cli_key_ec[]; +extern const char mbedtls_test_cli_pwd_ec[]; +extern const char mbedtls_test_cli_key_rsa[]; +extern const char mbedtls_test_cli_pwd_rsa[]; +extern const char mbedtls_test_cli_crt_rsa[]; + +extern const size_t mbedtls_test_cli_crt_ec_len; +extern const size_t mbedtls_test_cli_key_ec_len; +extern const size_t mbedtls_test_cli_pwd_ec_len; extern const size_t mbedtls_test_cli_key_rsa_len; -#endif +extern const size_t mbedtls_test_cli_pwd_rsa_len; +extern const size_t mbedtls_test_cli_crt_rsa_len; + +/* Config-dependent dispatch between EC and RSA + * (RSA if enabled, otherwise EC) */ + +extern const char * mbedtls_test_cli_crt; +extern const char * mbedtls_test_cli_key; +extern const char * mbedtls_test_cli_pwd; +extern const size_t mbedtls_test_cli_crt_len; +extern const size_t mbedtls_test_cli_key_len; +extern const size_t mbedtls_test_cli_pwd_len; #ifdef __cplusplus } diff --git a/thirdparty/mbedtls/include/mbedtls/cipher.h b/thirdparty/mbedtls/include/mbedtls/cipher.h index 922b6c32c6d..082a6917415 100644 --- a/thirdparty/mbedtls/include/mbedtls/cipher.h +++ b/thirdparty/mbedtls/include/mbedtls/cipher.h @@ -36,7 +36,7 @@ #endif #include -#include "mbedtls/platform_util.h" +#include "platform_util.h" #if defined(MBEDTLS_GCM_C) || defined(MBEDTLS_CCM_C) || defined(MBEDTLS_CHACHAPOLY_C) #define MBEDTLS_CIPHER_MODE_AEAD diff --git a/thirdparty/mbedtls/include/mbedtls/cmac.h b/thirdparty/mbedtls/include/mbedtls/cmac.h index c1967935319..9d42b3f209f 100644 --- a/thirdparty/mbedtls/include/mbedtls/cmac.h +++ b/thirdparty/mbedtls/include/mbedtls/cmac.h @@ -28,6 +28,12 @@ #ifndef MBEDTLS_CMAC_H #define MBEDTLS_CMAC_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "cipher.h" #ifdef __cplusplus diff --git a/thirdparty/mbedtls/include/mbedtls/compat-1.3.h b/thirdparty/mbedtls/include/mbedtls/compat-1.3.h index 213b6914035..a58b47243d9 100644 --- a/thirdparty/mbedtls/include/mbedtls/compat-1.3.h +++ b/thirdparty/mbedtls/include/mbedtls/compat-1.3.h @@ -25,6 +25,12 @@ * This file is part of mbed TLS (https://tls.mbed.org) */ +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #if ! defined(MBEDTLS_DEPRECATED_REMOVED) #if defined(MBEDTLS_DEPRECATED_WARNING) diff --git a/thirdparty/mbedtls/include/mbedtls/config.h b/thirdparty/mbedtls/include/mbedtls/config.h index 51d66291a5f..e16e1e53d38 100644 --- a/thirdparty/mbedtls/include/mbedtls/config.h +++ b/thirdparty/mbedtls/include/mbedtls/config.h @@ -686,6 +686,26 @@ */ #define MBEDTLS_REMOVE_ARC4_CIPHERSUITES +/** + * \def MBEDTLS_REMOVE_3DES_CIPHERSUITES + * + * Remove 3DES ciphersuites by default in SSL / TLS. + * This flag removes the ciphersuites based on 3DES from the default list as + * returned by mbedtls_ssl_list_ciphersuites(). However, it is still possible + * to enable (some of) them with mbedtls_ssl_conf_ciphersuites() by including + * them explicitly. + * + * A man-in-the-browser attacker can recover authentication tokens sent through + * a TLS connection using a 3DES based cipher suite (see "On the Practical + * (In-)Security of 64-bit Block Ciphers" by Karthikeyan Bhargavan and Gaëtan + * Leurent, see https://sweet32.info/SWEET32_CCS16.pdf). If this attack falls + * in your threat model or you are unsure, then you should keep this option + * enabled to remove 3DES based cipher suites. + * + * Comment this macro to keep 3DES in the default ciphersuite list. + */ +#define MBEDTLS_REMOVE_3DES_CIPHERSUITES + /** * \def MBEDTLS_ECP_DP_SECP192R1_ENABLED * @@ -1622,7 +1642,9 @@ * * Uncomment this to enable pthread mutexes. */ +// -- GODOT start -- //#define MBEDTLS_THREADING_PTHREAD +// -- GODOT end -- /** * \def MBEDTLS_VERSION_FEATURES @@ -2816,7 +2838,9 @@ * * Enable this layer to allow use of mutexes within mbed TLS */ +// -- GODOT start -- //#define MBEDTLS_THREADING_C +// -- GODOT end -- /** * \def MBEDTLS_TIMING_C diff --git a/thirdparty/mbedtls/include/mbedtls/ctr_drbg.h b/thirdparty/mbedtls/include/mbedtls/ctr_drbg.h index 10f9389d9f2..cc3df7b113f 100644 --- a/thirdparty/mbedtls/include/mbedtls/ctr_drbg.h +++ b/thirdparty/mbedtls/include/mbedtls/ctr_drbg.h @@ -36,6 +36,12 @@ #ifndef MBEDTLS_CTR_DRBG_H #define MBEDTLS_CTR_DRBG_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "aes.h" #if defined(MBEDTLS_THREADING_C) @@ -350,6 +356,8 @@ int mbedtls_ctr_drbg_write_seed_file( mbedtls_ctr_drbg_context *ctx, const char int mbedtls_ctr_drbg_update_seed_file( mbedtls_ctr_drbg_context *ctx, const char *path ); #endif /* MBEDTLS_FS_IO */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The CTR_DRBG checkup routine. * @@ -358,6 +366,8 @@ int mbedtls_ctr_drbg_update_seed_file( mbedtls_ctr_drbg_context *ctx, const char */ int mbedtls_ctr_drbg_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + /* Internal functions (do not call directly) */ int mbedtls_ctr_drbg_seed_entropy_len( mbedtls_ctr_drbg_context *, int (*)(void *, unsigned char *, size_t), void *, diff --git a/thirdparty/mbedtls/include/mbedtls/des.h b/thirdparty/mbedtls/include/mbedtls/des.h index d62042d14ef..54e6b7894b6 100644 --- a/thirdparty/mbedtls/include/mbedtls/des.h +++ b/thirdparty/mbedtls/include/mbedtls/des.h @@ -338,6 +338,8 @@ int mbedtls_des3_crypt_cbc( mbedtls_des3_context *ctx, void mbedtls_des_setkey( uint32_t SK[32], const unsigned char key[MBEDTLS_DES_KEY_SIZE] ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -345,6 +347,8 @@ void mbedtls_des_setkey( uint32_t SK[32], */ int mbedtls_des_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/dhm.h b/thirdparty/mbedtls/include/mbedtls/dhm.h index a5452c199ac..2909f5fbc8e 100644 --- a/thirdparty/mbedtls/include/mbedtls/dhm.h +++ b/thirdparty/mbedtls/include/mbedtls/dhm.h @@ -334,6 +334,8 @@ int mbedtls_dhm_parse_dhmfile( mbedtls_dhm_context *dhm, const char *path ); #endif /* MBEDTLS_FS_IO */ #endif /* MBEDTLS_ASN1_PARSE_C */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The DMH checkup routine. * @@ -342,6 +344,7 @@ int mbedtls_dhm_parse_dhmfile( mbedtls_dhm_context *dhm, const char *path ); */ int mbedtls_dhm_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/ecdh.h b/thirdparty/mbedtls/include/mbedtls/ecdh.h index 05b2b039700..4479a1d46fb 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecdh.h +++ b/thirdparty/mbedtls/include/mbedtls/ecdh.h @@ -34,6 +34,12 @@ #ifndef MBEDTLS_ECDH_H #define MBEDTLS_ECDH_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "ecp.h" /* diff --git a/thirdparty/mbedtls/include/mbedtls/ecdsa.h b/thirdparty/mbedtls/include/mbedtls/ecdsa.h index 40fdab3729f..f8b28507c21 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecdsa.h +++ b/thirdparty/mbedtls/include/mbedtls/ecdsa.h @@ -32,6 +32,12 @@ #ifndef MBEDTLS_ECDSA_H #define MBEDTLS_ECDSA_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "ecp.h" #include "md.h" diff --git a/thirdparty/mbedtls/include/mbedtls/ecjpake.h b/thirdparty/mbedtls/include/mbedtls/ecjpake.h index b967af83856..3d8d02ae646 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecjpake.h +++ b/thirdparty/mbedtls/include/mbedtls/ecjpake.h @@ -40,6 +40,11 @@ * The payloads are serialized in a way suitable for use in TLS, but could * also be use outside TLS. */ +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif #include "ecp.h" #include "md.h" diff --git a/thirdparty/mbedtls/include/mbedtls/ecp.h b/thirdparty/mbedtls/include/mbedtls/ecp.h index de3a343cb69..065a4cc0b99 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecp.h +++ b/thirdparty/mbedtls/include/mbedtls/ecp.h @@ -36,6 +36,12 @@ #ifndef MBEDTLS_ECP_H #define MBEDTLS_ECP_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "bignum.h" /* @@ -189,6 +195,68 @@ typedef struct mbedtls_ecp_group } mbedtls_ecp_group; +/** + * \name SECTION: Module settings + * + * The configuration options you can set for this module are in this section. + * Either change them in config.h, or define them using the compiler command line. + * \{ + */ + +#if !defined(MBEDTLS_ECP_MAX_BITS) +/** + * The maximum size of the groups, that is, of \c N and \c P. + */ +#define MBEDTLS_ECP_MAX_BITS 521 /**< The maximum size of groups, in bits. */ +#endif + +#define MBEDTLS_ECP_MAX_BYTES ( ( MBEDTLS_ECP_MAX_BITS + 7 ) / 8 ) +#define MBEDTLS_ECP_MAX_PT_LEN ( 2 * MBEDTLS_ECP_MAX_BYTES + 1 ) + +#if !defined(MBEDTLS_ECP_WINDOW_SIZE) +/* + * Maximum "window" size used for point multiplication. + * Default: 6. + * Minimum value: 2. Maximum value: 7. + * + * Result is an array of at most ( 1 << ( MBEDTLS_ECP_WINDOW_SIZE - 1 ) ) + * points used for point multiplication. This value is directly tied to EC + * peak memory usage, so decreasing it by one should roughly cut memory usage + * by two (if large curves are in use). + * + * Reduction in size may reduce speed, but larger curves are impacted first. + * Sample performances (in ECDHE handshakes/s, with FIXED_POINT_OPTIM = 1): + * w-size: 6 5 4 3 2 + * 521 145 141 135 120 97 + * 384 214 209 198 177 146 + * 256 320 320 303 262 226 + * 224 475 475 453 398 342 + * 192 640 640 633 587 476 + */ +#define MBEDTLS_ECP_WINDOW_SIZE 6 /**< The maximum window size used. */ +#endif /* MBEDTLS_ECP_WINDOW_SIZE */ + +#if !defined(MBEDTLS_ECP_FIXED_POINT_OPTIM) +/* + * Trade memory for speed on fixed-point multiplication. + * + * This speeds up repeated multiplication of the generator (that is, the + * multiplication in ECDSA signatures, and half of the multiplications in + * ECDSA verification and ECDHE) by a factor roughly 3 to 4. + * + * The cost is increasing EC peak memory usage by a factor roughly 2. + * + * Change this value to 0 to reduce peak memory usage. + */ +#define MBEDTLS_ECP_FIXED_POINT_OPTIM 1 /**< Enable fixed-point speed-up. */ +#endif /* MBEDTLS_ECP_FIXED_POINT_OPTIM */ + +/* \} name SECTION: Module settings */ + +#else /* MBEDTLS_ECP_ALT */ +#include "ecp_alt.h" +#endif /* MBEDTLS_ECP_ALT */ + #if defined(MBEDTLS_ECP_RESTARTABLE) /** @@ -253,68 +321,6 @@ typedef void mbedtls_ecp_restart_ctx; #endif /* MBEDTLS_ECP_RESTARTABLE */ -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in config.h, or define them using the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_ECP_MAX_BITS) -/** - * The maximum size of the groups, that is, of \c N and \c P. - */ -#define MBEDTLS_ECP_MAX_BITS 521 /**< The maximum size of groups, in bits. */ -#endif - -#define MBEDTLS_ECP_MAX_BYTES ( ( MBEDTLS_ECP_MAX_BITS + 7 ) / 8 ) -#define MBEDTLS_ECP_MAX_PT_LEN ( 2 * MBEDTLS_ECP_MAX_BYTES + 1 ) - -#if !defined(MBEDTLS_ECP_WINDOW_SIZE) -/* - * Maximum "window" size used for point multiplication. - * Default: 6. - * Minimum value: 2. Maximum value: 7. - * - * Result is an array of at most ( 1 << ( MBEDTLS_ECP_WINDOW_SIZE - 1 ) ) - * points used for point multiplication. This value is directly tied to EC - * peak memory usage, so decreasing it by one should roughly cut memory usage - * by two (if large curves are in use). - * - * Reduction in size may reduce speed, but larger curves are impacted first. - * Sample performances (in ECDHE handshakes/s, with FIXED_POINT_OPTIM = 1): - * w-size: 6 5 4 3 2 - * 521 145 141 135 120 97 - * 384 214 209 198 177 146 - * 256 320 320 303 262 226 - * 224 475 475 453 398 342 - * 192 640 640 633 587 476 - */ -#define MBEDTLS_ECP_WINDOW_SIZE 6 /**< The maximum window size used. */ -#endif /* MBEDTLS_ECP_WINDOW_SIZE */ - -#if !defined(MBEDTLS_ECP_FIXED_POINT_OPTIM) -/* - * Trade memory for speed on fixed-point multiplication. - * - * This speeds up repeated multiplication of the generator (that is, the - * multiplication in ECDSA signatures, and half of the multiplications in - * ECDSA verification and ECDHE) by a factor roughly 3 to 4. - * - * The cost is increasing EC peak memory usage by a factor roughly 2. - * - * Change this value to 0 to reduce peak memory usage. - */ -#define MBEDTLS_ECP_FIXED_POINT_OPTIM 1 /**< Enable fixed-point speed-up. */ -#endif /* MBEDTLS_ECP_FIXED_POINT_OPTIM */ - -/* \} name SECTION: Module settings */ - -#else /* MBEDTLS_ECP_ALT */ -#include "ecp_alt.h" -#endif /* MBEDTLS_ECP_ALT */ - /** * \brief The ECP key-pair structure. * @@ -476,7 +482,7 @@ void mbedtls_ecp_point_init( mbedtls_ecp_point *pt ); * * \note After this function is called, domain parameters * for various ECP groups can be loaded through the - * mbedtls_ecp_load() or mbedtls_ecp_tls_read_group() + * mbedtls_ecp_group_load() or mbedtls_ecp_tls_read_group() * functions. */ void mbedtls_ecp_group_init( mbedtls_ecp_group *grp ); diff --git a/thirdparty/mbedtls/include/mbedtls/ecp_internal.h b/thirdparty/mbedtls/include/mbedtls/ecp_internal.h index 18040697adf..7625ed48e1a 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecp_internal.h +++ b/thirdparty/mbedtls/include/mbedtls/ecp_internal.h @@ -61,6 +61,12 @@ #ifndef MBEDTLS_ECP_INTERNAL_H #define MBEDTLS_ECP_INTERNAL_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #if defined(MBEDTLS_ECP_INTERNAL_ALT) /** diff --git a/thirdparty/mbedtls/include/mbedtls/error.h b/thirdparty/mbedtls/include/mbedtls/error.h index 647a11a5667..bee0fe485ac 100644 --- a/thirdparty/mbedtls/include/mbedtls/error.h +++ b/thirdparty/mbedtls/include/mbedtls/error.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_ERROR_H #define MBEDTLS_ERROR_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include /** diff --git a/thirdparty/mbedtls/include/mbedtls/gcm.h b/thirdparty/mbedtls/include/mbedtls/gcm.h index fccabb0d973..fd130abd7cf 100644 --- a/thirdparty/mbedtls/include/mbedtls/gcm.h +++ b/thirdparty/mbedtls/include/mbedtls/gcm.h @@ -33,6 +33,12 @@ #ifndef MBEDTLS_GCM_H #define MBEDTLS_GCM_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "cipher.h" #include @@ -300,6 +306,8 @@ int mbedtls_gcm_finish( mbedtls_gcm_context *ctx, */ void mbedtls_gcm_free( mbedtls_gcm_context *ctx ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The GCM checkup routine. * @@ -308,6 +316,8 @@ void mbedtls_gcm_free( mbedtls_gcm_context *ctx ); */ int mbedtls_gcm_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/havege.h b/thirdparty/mbedtls/include/mbedtls/havege.h index 57e8c409431..4c1c86087a8 100644 --- a/thirdparty/mbedtls/include/mbedtls/havege.h +++ b/thirdparty/mbedtls/include/mbedtls/havege.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_HAVEGE_H #define MBEDTLS_HAVEGE_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include #define MBEDTLS_HAVEGE_COLLECT_SIZE 1024 diff --git a/thirdparty/mbedtls/include/mbedtls/hkdf.h b/thirdparty/mbedtls/include/mbedtls/hkdf.h index e6ed7cde97c..40ee64eb03c 100644 --- a/thirdparty/mbedtls/include/mbedtls/hkdf.h +++ b/thirdparty/mbedtls/include/mbedtls/hkdf.h @@ -27,6 +27,12 @@ #ifndef MBEDTLS_HKDF_H #define MBEDTLS_HKDF_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "md.h" /** diff --git a/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h b/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h index 146367b9dea..7eae32bbd67 100644 --- a/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h +++ b/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_HMAC_DRBG_H #define MBEDTLS_HMAC_DRBG_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "md.h" #if defined(MBEDTLS_THREADING_C) diff --git a/thirdparty/mbedtls/include/mbedtls/md2.h b/thirdparty/mbedtls/include/mbedtls/md2.h index f9bd98f8049..fe97cf08d40 100644 --- a/thirdparty/mbedtls/include/mbedtls/md2.h +++ b/thirdparty/mbedtls/include/mbedtls/md2.h @@ -283,6 +283,8 @@ MBEDTLS_DEPRECATED void mbedtls_md2( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -295,6 +297,8 @@ MBEDTLS_DEPRECATED void mbedtls_md2( const unsigned char *input, */ int mbedtls_md2_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/md4.h b/thirdparty/mbedtls/include/mbedtls/md4.h index dc3c0489491..ce703c0ba4f 100644 --- a/thirdparty/mbedtls/include/mbedtls/md4.h +++ b/thirdparty/mbedtls/include/mbedtls/md4.h @@ -288,6 +288,8 @@ MBEDTLS_DEPRECATED void mbedtls_md4( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -300,6 +302,8 @@ MBEDTLS_DEPRECATED void mbedtls_md4( const unsigned char *input, */ int mbedtls_md4_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/md5.h b/thirdparty/mbedtls/include/mbedtls/md5.h index 6c3354fd305..6eed6cc864a 100644 --- a/thirdparty/mbedtls/include/mbedtls/md5.h +++ b/thirdparty/mbedtls/include/mbedtls/md5.h @@ -288,6 +288,8 @@ MBEDTLS_DEPRECATED void mbedtls_md5( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -300,6 +302,8 @@ MBEDTLS_DEPRECATED void mbedtls_md5( const unsigned char *input, */ int mbedtls_md5_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/net.h b/thirdparty/mbedtls/include/mbedtls/net.h index 6c13b53fb93..8cead58e5df 100644 --- a/thirdparty/mbedtls/include/mbedtls/net.h +++ b/thirdparty/mbedtls/include/mbedtls/net.h @@ -23,6 +23,11 @@ * * This file is part of mbed TLS (https://tls.mbed.org) */ +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif #if !defined(MBEDTLS_DEPRECATED_REMOVED) #include "net_sockets.h" diff --git a/thirdparty/mbedtls/include/mbedtls/nist_kw.h b/thirdparty/mbedtls/include/mbedtls/nist_kw.h index 5a0f656a8f5..3b67b59cd29 100644 --- a/thirdparty/mbedtls/include/mbedtls/nist_kw.h +++ b/thirdparty/mbedtls/include/mbedtls/nist_kw.h @@ -37,6 +37,12 @@ #ifndef MBEDTLS_NIST_KW_H #define MBEDTLS_NIST_KW_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "cipher.h" #ifdef __cplusplus diff --git a/thirdparty/mbedtls/include/mbedtls/padlock.h b/thirdparty/mbedtls/include/mbedtls/padlock.h index 7a5d083a95b..721a5d4930e 100644 --- a/thirdparty/mbedtls/include/mbedtls/padlock.h +++ b/thirdparty/mbedtls/include/mbedtls/padlock.h @@ -28,6 +28,12 @@ #ifndef MBEDTLS_PADLOCK_H #define MBEDTLS_PADLOCK_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "aes.h" #define MBEDTLS_ERR_PADLOCK_DATA_MISALIGNED -0x0030 /**< Input data should be aligned. */ @@ -53,7 +59,7 @@ #define MBEDTLS_PADLOCK_PHE 0x0C00 #define MBEDTLS_PADLOCK_PMM 0x3000 -#define MBEDTLS_PADLOCK_ALIGN16(x) (uint32_t *) (16 + ((int32_t) x & ~15)) +#define MBEDTLS_PADLOCK_ALIGN16(x) (uint32_t *) (16 + ((int32_t) (x) & ~15)) #ifdef __cplusplus extern "C" { diff --git a/thirdparty/mbedtls/include/mbedtls/pem.h b/thirdparty/mbedtls/include/mbedtls/pem.h index fa82f7bdbd8..a29e9ce3002 100644 --- a/thirdparty/mbedtls/include/mbedtls/pem.h +++ b/thirdparty/mbedtls/include/mbedtls/pem.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_PEM_H #define MBEDTLS_PEM_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include /** diff --git a/thirdparty/mbedtls/include/mbedtls/pkcs12.h b/thirdparty/mbedtls/include/mbedtls/pkcs12.h index 69f04177c80..d441357b7f4 100644 --- a/thirdparty/mbedtls/include/mbedtls/pkcs12.h +++ b/thirdparty/mbedtls/include/mbedtls/pkcs12.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_PKCS12_H #define MBEDTLS_PKCS12_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "md.h" #include "cipher.h" #include "asn1.h" diff --git a/thirdparty/mbedtls/include/mbedtls/pkcs5.h b/thirdparty/mbedtls/include/mbedtls/pkcs5.h index d4bb36dfae4..c92185f7a68 100644 --- a/thirdparty/mbedtls/include/mbedtls/pkcs5.h +++ b/thirdparty/mbedtls/include/mbedtls/pkcs5.h @@ -26,6 +26,12 @@ #ifndef MBEDTLS_PKCS5_H #define MBEDTLS_PKCS5_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "asn1.h" #include "md.h" @@ -85,6 +91,8 @@ int mbedtls_pkcs5_pbkdf2_hmac( mbedtls_md_context_t *ctx, const unsigned char *p unsigned int iteration_count, uint32_t key_length, unsigned char *output ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -92,6 +100,8 @@ int mbedtls_pkcs5_pbkdf2_hmac( mbedtls_md_context_t *ctx, const unsigned char *p */ int mbedtls_pkcs5_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/platform_util.h b/thirdparty/mbedtls/include/mbedtls/platform_util.h index b0e72ad1496..dba6d459822 100644 --- a/thirdparty/mbedtls/include/mbedtls/platform_util.h +++ b/thirdparty/mbedtls/include/mbedtls/platform_util.h @@ -26,14 +26,14 @@ #define MBEDTLS_PLATFORM_UTIL_H #if !defined(MBEDTLS_CONFIG_FILE) -#include "mbedtls/config.h" +#include "config.h" #else #include MBEDTLS_CONFIG_FILE #endif #include #if defined(MBEDTLS_HAVE_TIME_DATE) -#include "mbedtls/platform_time.h" +#include "platform_time.h" #include #endif /* MBEDTLS_HAVE_TIME_DATE */ diff --git a/thirdparty/mbedtls/include/mbedtls/poly1305.h b/thirdparty/mbedtls/include/mbedtls/poly1305.h index 05866a2da61..f0ec44c9683 100644 --- a/thirdparty/mbedtls/include/mbedtls/poly1305.h +++ b/thirdparty/mbedtls/include/mbedtls/poly1305.h @@ -34,7 +34,7 @@ #define MBEDTLS_POLY1305_H #if !defined(MBEDTLS_CONFIG_FILE) -#include "mbedtls/config.h" +#include "config.h" #else #include MBEDTLS_CONFIG_FILE #endif diff --git a/thirdparty/mbedtls/include/mbedtls/ripemd160.h b/thirdparty/mbedtls/include/mbedtls/ripemd160.h index c74b7d2c6c8..b42f6d2a959 100644 --- a/thirdparty/mbedtls/include/mbedtls/ripemd160.h +++ b/thirdparty/mbedtls/include/mbedtls/ripemd160.h @@ -219,6 +219,8 @@ MBEDTLS_DEPRECATED void mbedtls_ripemd160( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -226,6 +228,8 @@ MBEDTLS_DEPRECATED void mbedtls_ripemd160( const unsigned char *input, */ int mbedtls_ripemd160_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/rsa.h b/thirdparty/mbedtls/include/mbedtls/rsa.h index ed65a34452b..906c4273325 100644 --- a/thirdparty/mbedtls/include/mbedtls/rsa.h +++ b/thirdparty/mbedtls/include/mbedtls/rsa.h @@ -1252,6 +1252,8 @@ int mbedtls_rsa_copy( mbedtls_rsa_context *dst, const mbedtls_rsa_context *src ) */ void mbedtls_rsa_free( mbedtls_rsa_context *ctx ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The RSA checkup routine. * @@ -1260,6 +1262,8 @@ void mbedtls_rsa_free( mbedtls_rsa_context *ctx ); */ int mbedtls_rsa_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/sha1.h b/thirdparty/mbedtls/include/mbedtls/sha1.h index 38ea10b1378..bb6ecf05a4a 100644 --- a/thirdparty/mbedtls/include/mbedtls/sha1.h +++ b/thirdparty/mbedtls/include/mbedtls/sha1.h @@ -328,6 +328,8 @@ MBEDTLS_DEPRECATED void mbedtls_sha1( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The SHA-1 checkup routine. * @@ -341,6 +343,8 @@ MBEDTLS_DEPRECATED void mbedtls_sha1( const unsigned char *input, */ int mbedtls_sha1_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/sha256.h b/thirdparty/mbedtls/include/mbedtls/sha256.h index 0e42f0abbaa..d64739820c6 100644 --- a/thirdparty/mbedtls/include/mbedtls/sha256.h +++ b/thirdparty/mbedtls/include/mbedtls/sha256.h @@ -278,6 +278,8 @@ MBEDTLS_DEPRECATED void mbedtls_sha256( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The SHA-224 and SHA-256 checkup routine. * @@ -286,6 +288,8 @@ MBEDTLS_DEPRECATED void mbedtls_sha256( const unsigned char *input, */ int mbedtls_sha256_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/include/mbedtls/sha512.h b/thirdparty/mbedtls/include/mbedtls/sha512.h index 7b26cf5cc3e..c06ceed1d13 100644 --- a/thirdparty/mbedtls/include/mbedtls/sha512.h +++ b/thirdparty/mbedtls/include/mbedtls/sha512.h @@ -253,6 +253,7 @@ int mbedtls_sha512_ret( const unsigned char *input, #else #define MBEDTLS_DEPRECATED #endif + /** * \brief This function calculates the SHA-512 or SHA-384 * checksum of a buffer. @@ -280,6 +281,9 @@ MBEDTLS_DEPRECATED void mbedtls_sha512( const unsigned char *input, #undef MBEDTLS_DEPRECATED #endif /* !MBEDTLS_DEPRECATED_REMOVED */ + +#if defined(MBEDTLS_SELF_TEST) + /** * \brief The SHA-384 or SHA-512 checkup routine. * @@ -287,6 +291,7 @@ MBEDTLS_DEPRECATED void mbedtls_sha512( const unsigned char *input, * \return \c 1 on failure. */ int mbedtls_sha512_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ #ifdef __cplusplus } diff --git a/thirdparty/mbedtls/include/mbedtls/ssl.h b/thirdparty/mbedtls/include/mbedtls/ssl.h index 8106bb4ab0f..d31f6cdd566 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl.h @@ -2532,22 +2532,28 @@ void mbedtls_ssl_conf_cert_req_ca_list( mbedtls_ssl_config *conf, #if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH) /** - * \brief Set the maximum fragment length to emit and/or negotiate - * (Default: the smaller of MBEDTLS_SSL_IN_CONTENT_LEN and - * MBEDTLS_SSL_OUT_CONTENT_LEN, usually 2^14 bytes) + * \brief Set the maximum fragment length to emit and/or negotiate. + * (Typical: the smaller of #MBEDTLS_SSL_IN_CONTENT_LEN and + * #MBEDTLS_SSL_OUT_CONTENT_LEN, usually `2^14` bytes) * (Server: set maximum fragment length to emit, - * usually negotiated by the client during handshake + * usually negotiated by the client during handshake) * (Client: set maximum fragment length to emit *and* * negotiate with the server during handshake) + * (Default: #MBEDTLS_SSL_MAX_FRAG_LEN_NONE) * - * \note With TLS, this currently only affects ApplicationData (sent - * with \c mbedtls_ssl_read()), not handshake messages. - * With DTLS, this affects both ApplicationData and handshake. + * \note On the client side, the maximum fragment length extension + * *will not* be used, unless the maximum fragment length has + * been set via this function to a value different than + * #MBEDTLS_SSL_MAX_FRAG_LEN_NONE. * * \note This sets the maximum length for a record's payload, * excluding record overhead that will be added to it, see * \c mbedtls_ssl_get_record_expansion(). * + * \note With TLS, this currently only affects ApplicationData (sent + * with \c mbedtls_ssl_read()), not handshake messages. + * With DTLS, this affects both ApplicationData and handshake. + * * \note For DTLS, it is also possible to set a limit for the total * size of daragrams passed to the transport layer, including * record overhead, see \c mbedtls_ssl_set_mtu(). diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_cache.h b/thirdparty/mbedtls/include/mbedtls/ssl_cache.h index ec081e6d244..52ba0948c55 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_cache.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_cache.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_SSL_CACHE_H #define MBEDTLS_SSL_CACHE_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "ssl.h" #if defined(MBEDTLS_THREADING_C) diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_ciphersuites.h b/thirdparty/mbedtls/include/mbedtls/ssl_ciphersuites.h index cda8b4835b6..71053e5ba71 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_ciphersuites.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_ciphersuites.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_SSL_CIPHERSUITES_H #define MBEDTLS_SSL_CIPHERSUITES_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "pk.h" #include "cipher.h" #include "md.h" diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_cookie.h b/thirdparty/mbedtls/include/mbedtls/ssl_cookie.h index 6a0ad4fa96d..e34760ae85e 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_cookie.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_cookie.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_SSL_COOKIE_H #define MBEDTLS_SSL_COOKIE_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "ssl.h" #if defined(MBEDTLS_THREADING_C) diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_internal.h b/thirdparty/mbedtls/include/mbedtls/ssl_internal.h index 97abb9f90b9..bd5ad94dbf0 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_internal.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_internal.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_SSL_INTERNAL_H #define MBEDTLS_SSL_INTERNAL_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + #include "ssl.h" #include "cipher.h" diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h b/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h index b2686df09f4..a84e7816e4a 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h @@ -24,6 +24,12 @@ #ifndef MBEDTLS_SSL_TICKET_H #define MBEDTLS_SSL_TICKET_H +#if !defined(MBEDTLS_CONFIG_FILE) +#include "config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + /* * This implementation of the session ticket callbacks includes key * management, rotating the keys periodically in order to preserve forward diff --git a/thirdparty/mbedtls/include/mbedtls/version.h b/thirdparty/mbedtls/include/mbedtls/version.h index 56e7398a2a2..ef8e4c1f4fe 100644 --- a/thirdparty/mbedtls/include/mbedtls/version.h +++ b/thirdparty/mbedtls/include/mbedtls/version.h @@ -40,16 +40,16 @@ */ #define MBEDTLS_VERSION_MAJOR 2 #define MBEDTLS_VERSION_MINOR 16 -#define MBEDTLS_VERSION_PATCH 0 +#define MBEDTLS_VERSION_PATCH 2 /** * The single version number has the following structure: * MMNNPP00 * Major version | Minor version | Patch version */ -#define MBEDTLS_VERSION_NUMBER 0x02100000 -#define MBEDTLS_VERSION_STRING "2.16.0" -#define MBEDTLS_VERSION_STRING_FULL "mbed TLS 2.16.0" +#define MBEDTLS_VERSION_NUMBER 0x02100200 +#define MBEDTLS_VERSION_STRING "2.16.2" +#define MBEDTLS_VERSION_STRING_FULL "mbed TLS 2.16.2" #if defined(MBEDTLS_VERSION_C) diff --git a/thirdparty/mbedtls/include/mbedtls/x509.h b/thirdparty/mbedtls/include/mbedtls/x509.h index d6db9c6e373..9ae825c1838 100644 --- a/thirdparty/mbedtls/include/mbedtls/x509.h +++ b/thirdparty/mbedtls/include/mbedtls/x509.h @@ -269,6 +269,8 @@ int mbedtls_x509_time_is_past( const mbedtls_x509_time *to ); */ int mbedtls_x509_time_is_future( const mbedtls_x509_time *from ); +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -276,6 +278,8 @@ int mbedtls_x509_time_is_future( const mbedtls_x509_time *from ); */ int mbedtls_x509_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + /* * Internal module functions. You probably do not want to use these unless you * know you do. diff --git a/thirdparty/mbedtls/include/mbedtls/x509_crt.h b/thirdparty/mbedtls/include/mbedtls/x509_crt.h index 3dd59224860..670bd10d891 100644 --- a/thirdparty/mbedtls/include/mbedtls/x509_crt.h +++ b/thirdparty/mbedtls/include/mbedtls/x509_crt.h @@ -98,7 +98,7 @@ mbedtls_x509_crt; * Build flag from an algorithm/curve identifier (pk, md, ecp) * Since 0 is always XXX_NONE, ignore it. */ -#define MBEDTLS_X509_ID_FLAG( id ) ( 1 << ( id - 1 ) ) +#define MBEDTLS_X509_ID_FLAG( id ) ( 1 << ( (id) - 1 ) ) /** * Security profile for certificate verification. diff --git a/thirdparty/mbedtls/include/mbedtls/x509_csr.h b/thirdparty/mbedtls/include/mbedtls/x509_csr.h index 0c6ccad78da..a3c28048e01 100644 --- a/thirdparty/mbedtls/include/mbedtls/x509_csr.h +++ b/thirdparty/mbedtls/include/mbedtls/x509_csr.h @@ -205,6 +205,14 @@ void mbedtls_x509write_csr_set_md_alg( mbedtls_x509write_csr *ctx, mbedtls_md_ty * \param key_usage key usage flags to set * * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED + * + * \note The decipherOnly flag from the Key Usage + * extension is represented by bit 8 (i.e. + * 0x8000), which cannot typically be represented + * in an unsigned char. Therefore, the flag + * decipherOnly (i.e. + * #MBEDTLS_X509_KU_DECIPHER_ONLY) cannot be set using this + * function. */ int mbedtls_x509write_csr_set_key_usage( mbedtls_x509write_csr *ctx, unsigned char key_usage ); diff --git a/thirdparty/mbedtls/include/mbedtls/xtea.h b/thirdparty/mbedtls/include/mbedtls/xtea.h index 6430c1318a5..b47f5535085 100644 --- a/thirdparty/mbedtls/include/mbedtls/xtea.h +++ b/thirdparty/mbedtls/include/mbedtls/xtea.h @@ -121,6 +121,8 @@ int mbedtls_xtea_crypt_cbc( mbedtls_xtea_context *ctx, unsigned char *output); #endif /* MBEDTLS_CIPHER_MODE_CBC */ +#if defined(MBEDTLS_SELF_TEST) + /** * \brief Checkup routine * @@ -128,6 +130,8 @@ int mbedtls_xtea_crypt_cbc( mbedtls_xtea_context *ctx, */ int mbedtls_xtea_self_test( int verbose ); +#endif /* MBEDTLS_SELF_TEST */ + #ifdef __cplusplus } #endif diff --git a/thirdparty/mbedtls/library/aes.c b/thirdparty/mbedtls/library/aes.c index 0543cd7817c..aff0a9939a8 100644 --- a/thirdparty/mbedtls/library/aes.c +++ b/thirdparty/mbedtls/library/aes.c @@ -395,9 +395,9 @@ static uint32_t RCON[10]; /* * Tables generation code */ -#define ROTL8(x) ( ( x << 8 ) & 0xFFFFFFFF ) | ( x >> 24 ) -#define XTIME(x) ( ( x << 1 ) ^ ( ( x & 0x80 ) ? 0x1B : 0x00 ) ) -#define MUL(x,y) ( ( x && y ) ? pow[(log[x]+log[y]) % 255] : 0 ) +#define ROTL8(x) ( ( (x) << 8 ) & 0xFFFFFFFF ) | ( (x) >> 24 ) +#define XTIME(x) ( ( (x) << 1 ) ^ ( ( (x) & 0x80 ) ? 0x1B : 0x00 ) ) +#define MUL(x,y) ( ( (x) && (y) ) ? pow[(log[(x)]+log[(y)]) % 255] : 0 ) static int aes_init_done = 0; @@ -815,51 +815,53 @@ int mbedtls_aes_xts_setkey_dec( mbedtls_aes_xts_context *ctx, #endif /* !MBEDTLS_AES_SETKEY_DEC_ALT */ -#define AES_FROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ -{ \ - X0 = *RK++ ^ AES_FT0( ( Y0 ) & 0xFF ) ^ \ - AES_FT1( ( Y1 >> 8 ) & 0xFF ) ^ \ - AES_FT2( ( Y2 >> 16 ) & 0xFF ) ^ \ - AES_FT3( ( Y3 >> 24 ) & 0xFF ); \ - \ - X1 = *RK++ ^ AES_FT0( ( Y1 ) & 0xFF ) ^ \ - AES_FT1( ( Y2 >> 8 ) & 0xFF ) ^ \ - AES_FT2( ( Y3 >> 16 ) & 0xFF ) ^ \ - AES_FT3( ( Y0 >> 24 ) & 0xFF ); \ - \ - X2 = *RK++ ^ AES_FT0( ( Y2 ) & 0xFF ) ^ \ - AES_FT1( ( Y3 >> 8 ) & 0xFF ) ^ \ - AES_FT2( ( Y0 >> 16 ) & 0xFF ) ^ \ - AES_FT3( ( Y1 >> 24 ) & 0xFF ); \ - \ - X3 = *RK++ ^ AES_FT0( ( Y3 ) & 0xFF ) ^ \ - AES_FT1( ( Y0 >> 8 ) & 0xFF ) ^ \ - AES_FT2( ( Y1 >> 16 ) & 0xFF ) ^ \ - AES_FT3( ( Y2 >> 24 ) & 0xFF ); \ -} +#define AES_FROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ + do \ + { \ + (X0) = *RK++ ^ AES_FT0( ( (Y0) ) & 0xFF ) ^ \ + AES_FT1( ( (Y1) >> 8 ) & 0xFF ) ^ \ + AES_FT2( ( (Y2) >> 16 ) & 0xFF ) ^ \ + AES_FT3( ( (Y3) >> 24 ) & 0xFF ); \ + \ + (X1) = *RK++ ^ AES_FT0( ( (Y1) ) & 0xFF ) ^ \ + AES_FT1( ( (Y2) >> 8 ) & 0xFF ) ^ \ + AES_FT2( ( (Y3) >> 16 ) & 0xFF ) ^ \ + AES_FT3( ( (Y0) >> 24 ) & 0xFF ); \ + \ + (X2) = *RK++ ^ AES_FT0( ( (Y2) ) & 0xFF ) ^ \ + AES_FT1( ( (Y3) >> 8 ) & 0xFF ) ^ \ + AES_FT2( ( (Y0) >> 16 ) & 0xFF ) ^ \ + AES_FT3( ( (Y1) >> 24 ) & 0xFF ); \ + \ + (X3) = *RK++ ^ AES_FT0( ( (Y3) ) & 0xFF ) ^ \ + AES_FT1( ( (Y0) >> 8 ) & 0xFF ) ^ \ + AES_FT2( ( (Y1) >> 16 ) & 0xFF ) ^ \ + AES_FT3( ( (Y2) >> 24 ) & 0xFF ); \ + } while( 0 ) -#define AES_RROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ -{ \ - X0 = *RK++ ^ AES_RT0( ( Y0 ) & 0xFF ) ^ \ - AES_RT1( ( Y3 >> 8 ) & 0xFF ) ^ \ - AES_RT2( ( Y2 >> 16 ) & 0xFF ) ^ \ - AES_RT3( ( Y1 >> 24 ) & 0xFF ); \ - \ - X1 = *RK++ ^ AES_RT0( ( Y1 ) & 0xFF ) ^ \ - AES_RT1( ( Y0 >> 8 ) & 0xFF ) ^ \ - AES_RT2( ( Y3 >> 16 ) & 0xFF ) ^ \ - AES_RT3( ( Y2 >> 24 ) & 0xFF ); \ - \ - X2 = *RK++ ^ AES_RT0( ( Y2 ) & 0xFF ) ^ \ - AES_RT1( ( Y1 >> 8 ) & 0xFF ) ^ \ - AES_RT2( ( Y0 >> 16 ) & 0xFF ) ^ \ - AES_RT3( ( Y3 >> 24 ) & 0xFF ); \ - \ - X3 = *RK++ ^ AES_RT0( ( Y3 ) & 0xFF ) ^ \ - AES_RT1( ( Y2 >> 8 ) & 0xFF ) ^ \ - AES_RT2( ( Y1 >> 16 ) & 0xFF ) ^ \ - AES_RT3( ( Y0 >> 24 ) & 0xFF ); \ -} +#define AES_RROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ + do \ + { \ + (X0) = *RK++ ^ AES_RT0( ( (Y0) ) & 0xFF ) ^ \ + AES_RT1( ( (Y3) >> 8 ) & 0xFF ) ^ \ + AES_RT2( ( (Y2) >> 16 ) & 0xFF ) ^ \ + AES_RT3( ( (Y1) >> 24 ) & 0xFF ); \ + \ + (X1) = *RK++ ^ AES_RT0( ( (Y1) ) & 0xFF ) ^ \ + AES_RT1( ( (Y0) >> 8 ) & 0xFF ) ^ \ + AES_RT2( ( (Y3) >> 16 ) & 0xFF ) ^ \ + AES_RT3( ( (Y2) >> 24 ) & 0xFF ); \ + \ + (X2) = *RK++ ^ AES_RT0( ( (Y2) ) & 0xFF ) ^ \ + AES_RT1( ( (Y1) >> 8 ) & 0xFF ) ^ \ + AES_RT2( ( (Y0) >> 16 ) & 0xFF ) ^ \ + AES_RT3( ( (Y3) >> 24 ) & 0xFF ); \ + \ + (X3) = *RK++ ^ AES_RT0( ( (Y3) ) & 0xFF ) ^ \ + AES_RT1( ( (Y2) >> 8 ) & 0xFF ) ^ \ + AES_RT2( ( (Y1) >> 16 ) & 0xFF ) ^ \ + AES_RT3( ( (Y0) >> 24 ) & 0xFF ); \ + } while( 0 ) /* * AES-ECB block encryption diff --git a/thirdparty/mbedtls/library/asn1write.c b/thirdparty/mbedtls/library/asn1write.c index a4d23f61967..c0b4622d58d 100644 --- a/thirdparty/mbedtls/library/asn1write.c +++ b/thirdparty/mbedtls/library/asn1write.c @@ -294,22 +294,28 @@ int mbedtls_asn1_write_bitstring( unsigned char **p, unsigned char *start, const unsigned char *buf, size_t bits ) { int ret; - size_t len = 0, size; + size_t len = 0; + size_t unused_bits, byte_len; - size = ( bits / 8 ) + ( ( bits % 8 ) ? 1 : 0 ); + byte_len = ( bits + 7 ) / 8; + unused_bits = ( byte_len * 8 ) - bits; - // Calculate byte length - // - if( *p < start || (size_t)( *p - start ) < size + 1 ) + if( *p < start || (size_t)( *p - start ) < byte_len + 1 ) return( MBEDTLS_ERR_ASN1_BUF_TOO_SMALL ); - len = size + 1; - (*p) -= size; - memcpy( *p, buf, size ); + len = byte_len + 1; - // Write unused bits - // - *--(*p) = (unsigned char) (size * 8 - bits); + /* Write the bitstring. Ensure the unused bits are zeroed */ + if( byte_len > 0 ) + { + byte_len--; + *--( *p ) = buf[byte_len] & ~( ( 0x1 << unused_bits ) - 1 ); + ( *p ) -= byte_len; + memcpy( *p, buf, byte_len ); + } + + /* Write unused bits */ + *--( *p ) = (unsigned char)unused_bits; MBEDTLS_ASN1_CHK_ADD( len, mbedtls_asn1_write_len( p, start, len ) ); MBEDTLS_ASN1_CHK_ADD( len, mbedtls_asn1_write_tag( p, start, MBEDTLS_ASN1_BIT_STRING ) ); diff --git a/thirdparty/mbedtls/library/bignum.c b/thirdparty/mbedtls/library/bignum.c index f968a0ad7d2..41946183c57 100644 --- a/thirdparty/mbedtls/library/bignum.c +++ b/thirdparty/mbedtls/library/bignum.c @@ -527,26 +527,38 @@ cleanup: } /* - * Helper to write the digits high-order first + * Helper to write the digits high-order first. */ -static int mpi_write_hlp( mbedtls_mpi *X, int radix, char **p ) +static int mpi_write_hlp( mbedtls_mpi *X, int radix, + char **p, const size_t buflen ) { int ret; mbedtls_mpi_uint r; + size_t length = 0; + char *p_end = *p + buflen; - if( radix < 2 || radix > 16 ) - return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); + do + { + if( length >= buflen ) + { + return( MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL ); + } - MBEDTLS_MPI_CHK( mbedtls_mpi_mod_int( &r, X, radix ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_div_int( X, NULL, X, radix ) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_mod_int( &r, X, radix ) ); + MBEDTLS_MPI_CHK( mbedtls_mpi_div_int( X, NULL, X, radix ) ); + /* + * Write the residue in the current position, as an ASCII character. + */ + if( r < 0xA ) + *(--p_end) = (char)( '0' + r ); + else + *(--p_end) = (char)( 'A' + ( r - 0xA ) ); - if( mbedtls_mpi_cmp_int( X, 0 ) != 0 ) - MBEDTLS_MPI_CHK( mpi_write_hlp( X, radix, p ) ); + length++; + } while( mbedtls_mpi_cmp_int( X, 0 ) != 0 ); - if( r < 10 ) - *(*p)++ = (char)( r + 0x30 ); - else - *(*p)++ = (char)( r + 0x37 ); + memmove( *p, p_end, length ); + *p += length; cleanup: @@ -570,15 +582,20 @@ int mbedtls_mpi_write_string( const mbedtls_mpi *X, int radix, if( radix < 2 || radix > 16 ) return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); - n = mbedtls_mpi_bitlen( X ); - if( radix >= 4 ) n >>= 1; - if( radix >= 16 ) n >>= 1; - /* - * Round up the buffer length to an even value to ensure that there is - * enough room for hexadecimal values that can be represented in an odd - * number of digits. - */ - n += 3 + ( ( n + 1 ) & 1 ); + n = mbedtls_mpi_bitlen( X ); /* Number of bits necessary to present `n`. */ + if( radix >= 4 ) n >>= 1; /* Number of 4-adic digits necessary to present + * `n`. If radix > 4, this might be a strict + * overapproximation of the number of + * radix-adic digits needed to present `n`. */ + if( radix >= 16 ) n >>= 1; /* Number of hexadecimal digits necessary to + * present `n`. */ + + n += 1; /* Terminating null byte */ + n += 1; /* Compensate for the divisions above, which round down `n` + * in case it's not even. */ + n += 1; /* Potential '-'-sign. */ + n += ( n & 1 ); /* Make n even to have enough space for hexadecimal writing, + * which always uses an even number of hex-digits. */ if( buflen < n ) { @@ -590,7 +607,10 @@ int mbedtls_mpi_write_string( const mbedtls_mpi *X, int radix, mbedtls_mpi_init( &T ); if( X->s == -1 ) + { *p++ = '-'; + buflen--; + } if( radix == 16 ) { @@ -619,7 +639,7 @@ int mbedtls_mpi_write_string( const mbedtls_mpi *X, int radix, if( T.s == -1 ) T.s = 1; - MBEDTLS_MPI_CHK( mpi_write_hlp( &T, radix, &p ) ); + MBEDTLS_MPI_CHK( mpi_write_hlp( &T, radix, &p, buflen ) ); } *p++ = '\0'; @@ -715,14 +735,101 @@ cleanup: } #endif /* MBEDTLS_FS_IO */ + +/* Convert a big-endian byte array aligned to the size of mbedtls_mpi_uint + * into the storage form used by mbedtls_mpi. */ + +static mbedtls_mpi_uint mpi_uint_bigendian_to_host_c( mbedtls_mpi_uint x ) +{ + uint8_t i; + mbedtls_mpi_uint tmp = 0; + /* This works regardless of the endianness. */ + for( i = 0; i < ciL; i++, x >>= 8 ) + tmp |= ( x & 0xFF ) << ( ( ciL - 1 - i ) << 3 ); + return( tmp ); +} + +static mbedtls_mpi_uint mpi_uint_bigendian_to_host( mbedtls_mpi_uint x ) +{ +#if defined(__BYTE_ORDER__) + +/* Nothing to do on bigendian systems. */ +#if ( __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ) + return( x ); +#endif /* __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ */ + +#if ( __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ) + +/* For GCC and Clang, have builtins for byte swapping. */ +#if defined(__GNUC__) && defined(__GNUC_PREREQ) +#if __GNUC_PREREQ(4,3) +#define have_bswap +#endif +#endif + +#if defined(__clang__) && defined(__has_builtin) +#if __has_builtin(__builtin_bswap32) && \ + __has_builtin(__builtin_bswap64) +#define have_bswap +#endif +#endif + +#if defined(have_bswap) + /* The compiler is hopefully able to statically evaluate this! */ + switch( sizeof(mbedtls_mpi_uint) ) + { + case 4: + return( __builtin_bswap32(x) ); + case 8: + return( __builtin_bswap64(x) ); + } +#endif +#endif /* __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ */ +#endif /* __BYTE_ORDER__ */ + + /* Fall back to C-based reordering if we don't know the byte order + * or we couldn't use a compiler-specific builtin. */ + return( mpi_uint_bigendian_to_host_c( x ) ); +} + +static void mpi_bigendian_to_host( mbedtls_mpi_uint * const p, size_t limbs ) +{ + mbedtls_mpi_uint *cur_limb_left; + mbedtls_mpi_uint *cur_limb_right; + if( limbs == 0 ) + return; + + /* + * Traverse limbs and + * - adapt byte-order in each limb + * - swap the limbs themselves. + * For that, simultaneously traverse the limbs from left to right + * and from right to left, as long as the left index is not bigger + * than the right index (it's not a problem if limbs is odd and the + * indices coincide in the last iteration). + */ + for( cur_limb_left = p, cur_limb_right = p + ( limbs - 1 ); + cur_limb_left <= cur_limb_right; + cur_limb_left++, cur_limb_right-- ) + { + mbedtls_mpi_uint tmp; + /* Note that if cur_limb_left == cur_limb_right, + * this code effectively swaps the bytes only once. */ + tmp = mpi_uint_bigendian_to_host( *cur_limb_left ); + *cur_limb_left = mpi_uint_bigendian_to_host( *cur_limb_right ); + *cur_limb_right = tmp; + } +} + /* * Import X from unsigned binary data, big endian */ int mbedtls_mpi_read_binary( mbedtls_mpi *X, const unsigned char *buf, size_t buflen ) { int ret; - size_t i, j; - size_t const limbs = CHARS_TO_LIMBS( buflen ); + size_t const limbs = CHARS_TO_LIMBS( buflen ); + size_t const overhead = ( limbs * ciL ) - buflen; + unsigned char *Xp; MPI_VALIDATE_RET( X != NULL ); MPI_VALIDATE_RET( buflen == 0 || buf != NULL ); @@ -734,11 +841,17 @@ int mbedtls_mpi_read_binary( mbedtls_mpi *X, const unsigned char *buf, size_t bu mbedtls_mpi_init( X ); MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, limbs ) ); } - MBEDTLS_MPI_CHK( mbedtls_mpi_lset( X, 0 ) ); - for( i = buflen, j = 0; i > 0; i--, j++ ) - X->p[j / ciL] |= ((mbedtls_mpi_uint) buf[i - 1]) << ((j % ciL) << 3); + /* Avoid calling `memcpy` with NULL source argument, + * even if buflen is 0. */ + if( buf != NULL ) + { + Xp = (unsigned char*) X->p; + memcpy( Xp + overhead, buf, buflen ); + + mpi_bigendian_to_host( X->p, limbs ); + } cleanup: @@ -1764,8 +1877,10 @@ int mbedtls_mpi_exp_mod( mbedtls_mpi *X, const mbedtls_mpi *A, wsize = ( i > 671 ) ? 6 : ( i > 239 ) ? 5 : ( i > 79 ) ? 4 : ( i > 23 ) ? 3 : 1; +#if( MBEDTLS_MPI_WINDOW_SIZE < 6 ) if( wsize > MBEDTLS_MPI_WINDOW_SIZE ) wsize = MBEDTLS_MPI_WINDOW_SIZE; +#endif j = N->n + 1; MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, j ) ); @@ -2008,18 +2123,28 @@ int mbedtls_mpi_fill_random( mbedtls_mpi *X, size_t size, void *p_rng ) { int ret; - unsigned char buf[MBEDTLS_MPI_MAX_SIZE]; + size_t const limbs = CHARS_TO_LIMBS( size ); + size_t const overhead = ( limbs * ciL ) - size; + unsigned char *Xp; + MPI_VALIDATE_RET( X != NULL ); MPI_VALIDATE_RET( f_rng != NULL ); - if( size > MBEDTLS_MPI_MAX_SIZE ) - return( MBEDTLS_ERR_MPI_BAD_INPUT_DATA ); + /* Ensure that target MPI has exactly the necessary number of limbs */ + if( X->n != limbs ) + { + mbedtls_mpi_free( X ); + mbedtls_mpi_init( X ); + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( X, limbs ) ); + } + MBEDTLS_MPI_CHK( mbedtls_mpi_lset( X, 0 ) ); - MBEDTLS_MPI_CHK( f_rng( p_rng, buf, size ) ); - MBEDTLS_MPI_CHK( mbedtls_mpi_read_binary( X, buf, size ) ); + Xp = (unsigned char*) X->p; + f_rng( p_rng, Xp + overhead, size ); + + mpi_bigendian_to_host( X->p, limbs ); cleanup: - mbedtls_platform_zeroize( buf, sizeof( buf ) ); return( ret ); } diff --git a/thirdparty/mbedtls/library/ccm.c b/thirdparty/mbedtls/library/ccm.c index 01e58b04361..c6211ee773c 100644 --- a/thirdparty/mbedtls/library/ccm.c +++ b/thirdparty/mbedtls/library/ccm.c @@ -134,11 +134,17 @@ void mbedtls_ccm_free( mbedtls_ccm_context *ctx ) * This avoids allocating one more 16 bytes buffer while allowing src == dst. */ #define CTR_CRYPT( dst, src, len ) \ - if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctr, 16, b, &olen ) ) != 0 ) \ - return( ret ); \ - \ - for( i = 0; i < len; i++ ) \ - dst[i] = src[i] ^ b[i]; + do \ + { \ + if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, ctr, \ + 16, b, &olen ) ) != 0 ) \ + { \ + return( ret ); \ + } \ + \ + for( i = 0; i < (len); i++ ) \ + (dst)[i] = (src)[i] ^ b[i]; \ + } while( 0 ) /* * Authenticated encryption or decryption diff --git a/thirdparty/mbedtls/library/certs.c b/thirdparty/mbedtls/library/certs.c index ff0f11e923c..b07fd8a3a1f 100644 --- a/thirdparty/mbedtls/library/certs.c +++ b/thirdparty/mbedtls/library/certs.c @@ -29,328 +29,1673 @@ #if defined(MBEDTLS_CERTS_C) -#if defined(MBEDTLS_ECDSA_C) -#define TEST_CA_CRT_EC \ -"-----BEGIN CERTIFICATE-----\r\n" \ -"MIICUjCCAdegAwIBAgIJAMFD4n5iQ8zoMAoGCCqGSM49BAMCMD4xCzAJBgNVBAYT\r\n" \ -"Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ -"QyBDQTAeFw0xMzA5MjQxNTQ5NDhaFw0yMzA5MjIxNTQ5NDhaMD4xCzAJBgNVBAYT\r\n" \ -"Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ -"QyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABMPaKzRBN1gvh1b+/Im6KUNLTuBu\r\n" \ -"ww5XUzM5WNRStJGVOQsj318XJGJI/BqVKc4sLYfCiFKAr9ZqqyHduNMcbli4yuiy\r\n" \ -"aY7zQa0pw7RfdadHb9UZKVVpmlM7ILRmFmAzHqOBoDCBnTAdBgNVHQ4EFgQUnW0g\r\n" \ -"JEkBPyvLeLUZvH4kydv7NnwwbgYDVR0jBGcwZYAUnW0gJEkBPyvLeLUZvH4kydv7\r\n" \ -"NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UE\r\n" \ -"AxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAwGA1UdEwQFMAMBAf8w\r\n" \ -"CgYIKoZIzj0EAwIDaQAwZgIxAMO0YnNWKJUAfXgSJtJxexn4ipg+kv4znuR50v56\r\n" \ -"t4d0PCu412mUC6Nnd7izvtE2MgIxAP1nnJQjZ8BWukszFQDG48wxCCyci9qpdSMv\r\n" \ -"uCjn8pwUOkABXK8Mss90fzCfCEOtIA==\r\n" \ -"-----END CERTIFICATE-----\r\n" -const char mbedtls_test_ca_crt_ec[] = TEST_CA_CRT_EC; -const size_t mbedtls_test_ca_crt_ec_len = sizeof( mbedtls_test_ca_crt_ec ); +/* + * Test CA Certificates + * + * We define test CA certificates for each choice of the following parameters: + * - PEM or DER encoding + * - SHA-1 or SHA-256 hash + * - RSA or EC key + * + * Things to add: + * - multiple EC curve types + * + */ -const char mbedtls_test_ca_key_ec[] = -"-----BEGIN EC PRIVATE KEY-----\r\n" -"Proc-Type: 4,ENCRYPTED\r\n" -"DEK-Info: DES-EDE3-CBC,307EAB469933D64E\r\n" -"\r\n" -"IxbrRmKcAzctJqPdTQLA4SWyBYYGYJVkYEna+F7Pa5t5Yg/gKADrFKcm6B72e7DG\r\n" -"ihExtZI648s0zdYw6qSJ74vrPSuWDe5qm93BqsfVH9svtCzWHW0pm1p0KTBCFfUq\r\n" -"UsuWTITwJImcnlAs1gaRZ3sAWm7cOUidL0fo2G0fYUFNcYoCSLffCFTEHBuPnagb\r\n" -"a77x/sY1Bvii8S9/XhDTb6pTMx06wzrm\r\n" -"-----END EC PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_ca_key_ec_len = sizeof( mbedtls_test_ca_key_ec ); +/* This is taken from tests/data_files/test-ca2.crt */ +/* BEGIN FILE string macro TEST_CA_CRT_EC_PEM tests/data_files/test-ca2.crt */ +#define TEST_CA_CRT_EC_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIICUjCCAdegAwIBAgIJAMFD4n5iQ8zoMAoGCCqGSM49BAMCMD4xCzAJBgNVBAYT\r\n" \ + "Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ + "QyBDQTAeFw0xMzA5MjQxNTQ5NDhaFw0yMzA5MjIxNTQ5NDhaMD4xCzAJBgNVBAYT\r\n" \ + "Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ + "QyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABMPaKzRBN1gvh1b+/Im6KUNLTuBu\r\n" \ + "ww5XUzM5WNRStJGVOQsj318XJGJI/BqVKc4sLYfCiFKAr9ZqqyHduNMcbli4yuiy\r\n" \ + "aY7zQa0pw7RfdadHb9UZKVVpmlM7ILRmFmAzHqOBoDCBnTAdBgNVHQ4EFgQUnW0g\r\n" \ + "JEkBPyvLeLUZvH4kydv7NnwwbgYDVR0jBGcwZYAUnW0gJEkBPyvLeLUZvH4kydv7\r\n" \ + "NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UE\r\n" \ + "AxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAwGA1UdEwQFMAMBAf8w\r\n" \ + "CgYIKoZIzj0EAwIDaQAwZgIxAMO0YnNWKJUAfXgSJtJxexn4ipg+kv4znuR50v56\r\n" \ + "t4d0PCu412mUC6Nnd7izvtE2MgIxAP1nnJQjZ8BWukszFQDG48wxCCyci9qpdSMv\r\n" \ + "uCjn8pwUOkABXK8Mss90fzCfCEOtIA==\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ -const char mbedtls_test_ca_pwd_ec[] = "PolarSSLTest"; -const size_t mbedtls_test_ca_pwd_ec_len = sizeof( mbedtls_test_ca_pwd_ec ) - 1; +/* This is generated from tests/data_files/test-ca2.crt.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CA_CRT_EC_DER tests/data_files/test-ca2.crt.der */ +#define TEST_CA_CRT_EC_DER { \ + 0x30, 0x82, 0x02, 0x52, 0x30, 0x82, 0x01, 0xd7, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x09, 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, \ + 0x30, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, \ + 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, \ + 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, \ + 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, \ + 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, \ + 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, 0x31, 0x33, 0x30, 0x39, \ + 0x32, 0x34, 0x31, 0x35, 0x34, 0x39, 0x34, 0x38, 0x5a, 0x17, 0x0d, 0x32, \ + 0x33, 0x30, 0x39, 0x32, 0x32, 0x31, 0x35, 0x34, 0x39, 0x34, 0x38, 0x5a, \ + 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, \ + 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, \ + 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, \ + 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, \ + 0x43, 0x20, 0x43, 0x41, 0x30, 0x76, 0x30, 0x10, 0x06, 0x07, 0x2a, 0x86, \ + 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x22, \ + 0x03, 0x62, 0x00, 0x04, 0xc3, 0xda, 0x2b, 0x34, 0x41, 0x37, 0x58, 0x2f, \ + 0x87, 0x56, 0xfe, 0xfc, 0x89, 0xba, 0x29, 0x43, 0x4b, 0x4e, 0xe0, 0x6e, \ + 0xc3, 0x0e, 0x57, 0x53, 0x33, 0x39, 0x58, 0xd4, 0x52, 0xb4, 0x91, 0x95, \ + 0x39, 0x0b, 0x23, 0xdf, 0x5f, 0x17, 0x24, 0x62, 0x48, 0xfc, 0x1a, 0x95, \ + 0x29, 0xce, 0x2c, 0x2d, 0x87, 0xc2, 0x88, 0x52, 0x80, 0xaf, 0xd6, 0x6a, \ + 0xab, 0x21, 0xdd, 0xb8, 0xd3, 0x1c, 0x6e, 0x58, 0xb8, 0xca, 0xe8, 0xb2, \ + 0x69, 0x8e, 0xf3, 0x41, 0xad, 0x29, 0xc3, 0xb4, 0x5f, 0x75, 0xa7, 0x47, \ + 0x6f, 0xd5, 0x19, 0x29, 0x55, 0x69, 0x9a, 0x53, 0x3b, 0x20, 0xb4, 0x66, \ + 0x16, 0x60, 0x33, 0x1e, 0xa3, 0x81, 0xa0, 0x30, 0x81, 0x9d, 0x30, 0x1d, \ + 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x9d, 0x6d, 0x20, \ + 0x24, 0x49, 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, \ + 0xc9, 0xdb, 0xfb, 0x36, 0x7c, 0x30, 0x6e, 0x06, 0x03, 0x55, 0x1d, 0x23, \ + 0x04, 0x67, 0x30, 0x65, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, 0x01, \ + 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, 0xfb, \ + 0x36, 0x7c, 0xa1, 0x42, 0xa4, 0x40, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, \ + 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, \ + 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, \ + 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, \ + 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, \ + 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x82, 0x09, \ + 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, 0x30, 0x0c, 0x06, \ + 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, \ + 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, 0x03, \ + 0x69, 0x00, 0x30, 0x66, 0x02, 0x31, 0x00, 0xc3, 0xb4, 0x62, 0x73, 0x56, \ + 0x28, 0x95, 0x00, 0x7d, 0x78, 0x12, 0x26, 0xd2, 0x71, 0x7b, 0x19, 0xf8, \ + 0x8a, 0x98, 0x3e, 0x92, 0xfe, 0x33, 0x9e, 0xe4, 0x79, 0xd2, 0xfe, 0x7a, \ + 0xb7, 0x87, 0x74, 0x3c, 0x2b, 0xb8, 0xd7, 0x69, 0x94, 0x0b, 0xa3, 0x67, \ + 0x77, 0xb8, 0xb3, 0xbe, 0xd1, 0x36, 0x32, 0x02, 0x31, 0x00, 0xfd, 0x67, \ + 0x9c, 0x94, 0x23, 0x67, 0xc0, 0x56, 0xba, 0x4b, 0x33, 0x15, 0x00, 0xc6, \ + 0xe3, 0xcc, 0x31, 0x08, 0x2c, 0x9c, 0x8b, 0xda, 0xa9, 0x75, 0x23, 0x2f, \ + 0xb8, 0x28, 0xe7, 0xf2, 0x9c, 0x14, 0x3a, 0x40, 0x01, 0x5c, 0xaf, 0x0c, \ + 0xb2, 0xcf, 0x74, 0x7f, 0x30, 0x9f, 0x08, 0x43, 0xad, 0x20 \ +} +/* END FILE */ -const char mbedtls_test_srv_crt_ec[] = -"-----BEGIN CERTIFICATE-----\r\n" -"MIICHzCCAaWgAwIBAgIBCTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\r\n" -"MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UEChMIUG9sYXJTU0wxEjAQBgNVBAMTCWxvY2FsaG9zdDBZMBMGByqGSM49AgEG\r\n" -"CCqGSM49AwEHA0IABDfMVtl2CR5acj7HWS3/IG7ufPkGkXTQrRS192giWWKSTuUA\r\n" -"2CMR/+ov0jRdXRa9iojCa3cNVc2KKg76Aci07f+jgZ0wgZowCQYDVR0TBAIwADAd\r\n" -"BgNVHQ4EFgQUUGGlj9QH2deCAQzlZX+MY0anE74wbgYDVR0jBGcwZYAUnW0gJEkB\r\n" -"PyvLeLUZvH4kydv7NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xh\r\n" -"clNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAoG\r\n" -"CCqGSM49BAMCA2gAMGUCMQCaLFzXptui5WQN8LlO3ddh1hMxx6tzgLvT03MTVK2S\r\n" -"C12r0Lz3ri/moSEpNZWqPjkCMCE2f53GXcYLqyfyJR078c/xNSUU5+Xxl7VZ414V\r\n" -"fGa5kHvHARBPc8YAIVIqDvHH1Q==\r\n" -"-----END CERTIFICATE-----\r\n"; -const size_t mbedtls_test_srv_crt_ec_len = sizeof( mbedtls_test_srv_crt_ec ); +/* This is taken from tests/data_files/test-ca2.key.enc */ +/* BEGIN FILE string macro TEST_CA_KEY_EC_PEM tests/data_files/test-ca2.key.enc */ +#define TEST_CA_KEY_EC_PEM \ + "-----BEGIN EC PRIVATE KEY-----\r\n" \ + "Proc-Type: 4,ENCRYPTED\r\n" \ + "DEK-Info: DES-EDE3-CBC,307EAB469933D64E\r\n" \ + "\r\n" \ + "IxbrRmKcAzctJqPdTQLA4SWyBYYGYJVkYEna+F7Pa5t5Yg/gKADrFKcm6B72e7DG\r\n" \ + "ihExtZI648s0zdYw6qSJ74vrPSuWDe5qm93BqsfVH9svtCzWHW0pm1p0KTBCFfUq\r\n" \ + "UsuWTITwJImcnlAs1gaRZ3sAWm7cOUidL0fo2G0fYUFNcYoCSLffCFTEHBuPnagb\r\n" \ + "a77x/sY1Bvii8S9/XhDTb6pTMx06wzrm\r\n" \ + "-----END EC PRIVATE KEY-----\r\n" +/* END FILE */ -const char mbedtls_test_srv_key_ec[] = -"-----BEGIN EC PRIVATE KEY-----\r\n" -"MHcCAQEEIPEqEyB2AnCoPL/9U/YDHvdqXYbIogTywwyp6/UfDw6noAoGCCqGSM49\r\n" -"AwEHoUQDQgAEN8xW2XYJHlpyPsdZLf8gbu58+QaRdNCtFLX3aCJZYpJO5QDYIxH/\r\n" -"6i/SNF1dFr2KiMJrdw1VzYoqDvoByLTt/w==\r\n" -"-----END EC PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_srv_key_ec_len = sizeof( mbedtls_test_srv_key_ec ); +#define TEST_CA_PWD_EC_PEM "PolarSSLTest" -const char mbedtls_test_cli_crt_ec[] = -"-----BEGIN CERTIFICATE-----\r\n" -"MIICLDCCAbKgAwIBAgIBDTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\r\n" -"MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjBBMQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UEChMIUG9sYXJTU0wxHzAdBgNVBAMTFlBvbGFyU1NMIFRlc3QgQ2xpZW50IDIw\r\n" -"WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARX5a6xc9/TrLuTuIH/Eq7u5lOszlVT\r\n" -"9jQOzC7jYyUL35ji81xgNpbA1RgUcOV/n9VLRRjlsGzVXPiWj4dwo+THo4GdMIGa\r\n" -"MAkGA1UdEwQCMAAwHQYDVR0OBBYEFHoAX4Zk/OBd5REQO7LmO8QmP8/iMG4GA1Ud\r\n" -"IwRnMGWAFJ1tICRJAT8ry3i1Gbx+JMnb+zZ8oUKkQDA+MQswCQYDVQQGEwJOTDER\r\n" -"MA8GA1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0GC\r\n" -"CQDBQ+J+YkPM6DAKBggqhkjOPQQDAgNoADBlAjBKZQ17IIOimbmoD/yN7o89u3BM\r\n" -"lgOsjnhw3fIOoLIWy2WOGsk/LGF++DzvrRzuNiACMQCd8iem1XS4JK7haj8xocpU\r\n" -"LwjQje5PDGHfd3h9tP38Qknu5bJqws0md2KOKHyeV0U=\r\n" -"-----END CERTIFICATE-----\r\n"; -const size_t mbedtls_test_cli_crt_ec_len = sizeof( mbedtls_test_cli_crt_ec ); +/* This is generated from tests/data_files/test-ca2.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CA_KEY_EC_DER tests/data_files/test-ca2.key.der */ +#define TEST_CA_KEY_EC_DER { \ + 0x30, 0x81, 0xa4, 0x02, 0x01, 0x01, 0x04, 0x30, 0x83, 0xd9, 0x15, 0x0e, \ + 0xa0, 0x71, 0xf0, 0x57, 0x10, 0x33, 0xa3, 0x38, 0xb8, 0x86, 0xc1, 0xa6, \ + 0x11, 0x5d, 0x6d, 0xb4, 0x03, 0xe1, 0x29, 0x76, 0x45, 0xd7, 0x87, 0x6f, \ + 0x23, 0xab, 0x44, 0x20, 0xea, 0x64, 0x7b, 0x85, 0xb1, 0x76, 0xe7, 0x85, \ + 0x95, 0xaa, 0x74, 0xd6, 0xd1, 0xa4, 0x5e, 0xea, 0xa0, 0x07, 0x06, 0x05, \ + 0x2b, 0x81, 0x04, 0x00, 0x22, 0xa1, 0x64, 0x03, 0x62, 0x00, 0x04, 0xc3, \ + 0xda, 0x2b, 0x34, 0x41, 0x37, 0x58, 0x2f, 0x87, 0x56, 0xfe, 0xfc, 0x89, \ + 0xba, 0x29, 0x43, 0x4b, 0x4e, 0xe0, 0x6e, 0xc3, 0x0e, 0x57, 0x53, 0x33, \ + 0x39, 0x58, 0xd4, 0x52, 0xb4, 0x91, 0x95, 0x39, 0x0b, 0x23, 0xdf, 0x5f, \ + 0x17, 0x24, 0x62, 0x48, 0xfc, 0x1a, 0x95, 0x29, 0xce, 0x2c, 0x2d, 0x87, \ + 0xc2, 0x88, 0x52, 0x80, 0xaf, 0xd6, 0x6a, 0xab, 0x21, 0xdd, 0xb8, 0xd3, \ + 0x1c, 0x6e, 0x58, 0xb8, 0xca, 0xe8, 0xb2, 0x69, 0x8e, 0xf3, 0x41, 0xad, \ + 0x29, 0xc3, 0xb4, 0x5f, 0x75, 0xa7, 0x47, 0x6f, 0xd5, 0x19, 0x29, 0x55, \ + 0x69, 0x9a, 0x53, 0x3b, 0x20, 0xb4, 0x66, 0x16, 0x60, 0x33, 0x1e \ +} +/* END FILE */ -const char mbedtls_test_cli_key_ec[] = -"-----BEGIN EC PRIVATE KEY-----\r\n" -"MHcCAQEEIPb3hmTxZ3/mZI3vyk7p3U3wBf+WIop6hDhkFzJhmLcqoAoGCCqGSM49\r\n" -"AwEHoUQDQgAEV+WusXPf06y7k7iB/xKu7uZTrM5VU/Y0Dswu42MlC9+Y4vNcYDaW\r\n" -"wNUYFHDlf5/VS0UY5bBs1Vz4lo+HcKPkxw==\r\n" -"-----END EC PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_cli_key_ec_len = sizeof( mbedtls_test_cli_key_ec ); -#endif /* MBEDTLS_ECDSA_C */ +/* This is taken from tests/data_files/test-ca-sha256.crt. */ +/* BEGIN FILE string macro TEST_CA_CRT_RSA_SHA256_PEM tests/data_files/test-ca-sha256.crt */ +#define TEST_CA_CRT_RSA_SHA256_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIIDQTCCAimgAwIBAgIBAzANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ + "MTEwMjEyMTQ0NDAwWhcNMjEwMjEyMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ + "CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ + "mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ + "50BwFMtEonILwuVA+T7lpg6z+exKY8C4KQB0nFc7qKUEkHHxvYPZP9al4jwqj+8n\r\n" \ + "YMPGn8u67GB9t+aEMr5P+1gmIgNb1LTV+/Xjli5wwOQuvfwu7uJBVcA0Ln0kcmnL\r\n" \ + "R7EUQIN9Z/SG9jGr8XmksrUuEvmEF/Bibyc+E1ixVA0hmnM3oTDPb5Lc9un8rNsu\r\n" \ + "KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ + "UDBOMAwGA1UdEwQFMAMBAf8wHQYDVR0OBBYEFLRa5KWz3tJS9rnVppUP6z68x/3/\r\n" \ + "MB8GA1UdIwQYMBaAFLRa5KWz3tJS9rnVppUP6z68x/3/MA0GCSqGSIb3DQEBCwUA\r\n" \ + "A4IBAQB2W2dIy4q4KysbrTL4HIaOqu62RceGuQ/KhyiI6O0ndCtQ/PgCBqHHTP8u\r\n" \ + "8F1X2ivb60ynHV6baMLPI4Kf1k4MONtLSf/++1qh0Gdycd3A8IDAfy0YnC1F3OPK\r\n" \ + "vWO/cZGitKoTbEpP4y4Rng3sFCDndRCWIRIDOEEW/H3lCcfL7sOQojdLl85ajFkh\r\n" \ + "YvcDqjmnTcspUnuq9Y00C7porXJthZwz1S18qVjcFNk0zEhVMUbupSrdXVmKtOJW\r\n" \ + "MWZjgcA+OXzcnb2hSKWbhjykH/u6/PqkuHPkD723rwXbmHdxRVS9CW57kDkn5ezJ\r\n" \ + "5pE6Sam4qFsCNFJNBV9FRf3ZBMFi\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is generated from tests/data_files/test-ca-sha256.crt.der + * using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CA_CRT_RSA_SHA256_DER tests/data_files/test-ca-sha256.crt.der */ +#define TEST_CA_CRT_RSA_SHA256_DER { \ + 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ + 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ + 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ + 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ + 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ + 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ + 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ + 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ + 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ + 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ + 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ + 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ + 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ + 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ + 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ + 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ + 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ + 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ + 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ + 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ + 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ + 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ + 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ + 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ + 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ + 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ + 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ + 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ + 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ + 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x01, 0x00, 0x76, 0x5b, 0x67, 0x48, 0xcb, 0x8a, 0xb8, \ + 0x2b, 0x2b, 0x1b, 0xad, 0x32, 0xf8, 0x1c, 0x86, 0x8e, 0xaa, 0xee, 0xb6, \ + 0x45, 0xc7, 0x86, 0xb9, 0x0f, 0xca, 0x87, 0x28, 0x88, 0xe8, 0xed, 0x27, \ + 0x74, 0x2b, 0x50, 0xfc, 0xf8, 0x02, 0x06, 0xa1, 0xc7, 0x4c, 0xff, 0x2e, \ + 0xf0, 0x5d, 0x57, 0xda, 0x2b, 0xdb, 0xeb, 0x4c, 0xa7, 0x1d, 0x5e, 0x9b, \ + 0x68, 0xc2, 0xcf, 0x23, 0x82, 0x9f, 0xd6, 0x4e, 0x0c, 0x38, 0xdb, 0x4b, \ + 0x49, 0xff, 0xfe, 0xfb, 0x5a, 0xa1, 0xd0, 0x67, 0x72, 0x71, 0xdd, 0xc0, \ + 0xf0, 0x80, 0xc0, 0x7f, 0x2d, 0x18, 0x9c, 0x2d, 0x45, 0xdc, 0xe3, 0xca, \ + 0xbd, 0x63, 0xbf, 0x71, 0x91, 0xa2, 0xb4, 0xaa, 0x13, 0x6c, 0x4a, 0x4f, \ + 0xe3, 0x2e, 0x11, 0x9e, 0x0d, 0xec, 0x14, 0x20, 0xe7, 0x75, 0x10, 0x96, \ + 0x21, 0x12, 0x03, 0x38, 0x41, 0x16, 0xfc, 0x7d, 0xe5, 0x09, 0xc7, 0xcb, \ + 0xee, 0xc3, 0x90, 0xa2, 0x37, 0x4b, 0x97, 0xce, 0x5a, 0x8c, 0x59, 0x21, \ + 0x62, 0xf7, 0x03, 0xaa, 0x39, 0xa7, 0x4d, 0xcb, 0x29, 0x52, 0x7b, 0xaa, \ + 0xf5, 0x8d, 0x34, 0x0b, 0xba, 0x68, 0xad, 0x72, 0x6d, 0x85, 0x9c, 0x33, \ + 0xd5, 0x2d, 0x7c, 0xa9, 0x58, 0xdc, 0x14, 0xd9, 0x34, 0xcc, 0x48, 0x55, \ + 0x31, 0x46, 0xee, 0xa5, 0x2a, 0xdd, 0x5d, 0x59, 0x8a, 0xb4, 0xe2, 0x56, \ + 0x31, 0x66, 0x63, 0x81, 0xc0, 0x3e, 0x39, 0x7c, 0xdc, 0x9d, 0xbd, 0xa1, \ + 0x48, 0xa5, 0x9b, 0x86, 0x3c, 0xa4, 0x1f, 0xfb, 0xba, 0xfc, 0xfa, 0xa4, \ + 0xb8, 0x73, 0xe4, 0x0f, 0xbd, 0xb7, 0xaf, 0x05, 0xdb, 0x98, 0x77, 0x71, \ + 0x45, 0x54, 0xbd, 0x09, 0x6e, 0x7b, 0x90, 0x39, 0x27, 0xe5, 0xec, 0xc9, \ + 0xe6, 0x91, 0x3a, 0x49, 0xa9, 0xb8, 0xa8, 0x5b, 0x02, 0x34, 0x52, 0x4d, \ + 0x05, 0x5f, 0x45, 0x45, 0xfd, 0xd9, 0x04, 0xc1, 0x62 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/test-ca-sha1.crt. */ +/* BEGIN FILE string macro TEST_CA_CRT_RSA_SHA1_PEM tests/data_files/test-ca-sha1.crt */ +#define TEST_CA_CRT_RSA_SHA1_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIIDQTCCAimgAwIBAgIBAzANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ + "MTEwMjEyMTQ0NDAwWhcNMjEwMjEyMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ + "CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ + "mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ + "50BwFMtEonILwuVA+T7lpg6z+exKY8C4KQB0nFc7qKUEkHHxvYPZP9al4jwqj+8n\r\n" \ + "YMPGn8u67GB9t+aEMr5P+1gmIgNb1LTV+/Xjli5wwOQuvfwu7uJBVcA0Ln0kcmnL\r\n" \ + "R7EUQIN9Z/SG9jGr8XmksrUuEvmEF/Bibyc+E1ixVA0hmnM3oTDPb5Lc9un8rNsu\r\n" \ + "KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ + "UDBOMAwGA1UdEwQFMAMBAf8wHQYDVR0OBBYEFLRa5KWz3tJS9rnVppUP6z68x/3/\r\n" \ + "MB8GA1UdIwQYMBaAFLRa5KWz3tJS9rnVppUP6z68x/3/MA0GCSqGSIb3DQEBBQUA\r\n" \ + "A4IBAQABE3OEPfEd/bcJW5ZdU3/VgPNS4tMzh8gnJP/V2FcvFtGylMpQq6YnEBYI\r\n" \ + "yBHAL4DRvlMY5rnXGBp3ODR8MpqHC6AquRTCLzjS57iYff//4QFQqW9n92zctspv\r\n" \ + "czkaPKgjqo1No3Uq0Xaz10rcxyTUPrf5wNVRZ2V0KvllvAAVSzbI4mpdUXztjhST\r\n" \ + "S5A2BeWQAAOr0zq1F7TSRVJpJs7jmB2ai/igkh1IAjcuwV6VwlP+sbw0gjQ0NpGM\r\n" \ + "iHpnlzRAi/tIbtOvMIGOBU2TIfax/5jq1agUx5aPmT5TWAiJPOOP6l5xXnDwxeYS\r\n" \ + "NWqiX9GyusBZjezaCaHabjDLU0qQ\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is taken from tests/data_files/test-ca-sha1.crt.der. */ +/* BEGIN FILE binary macro TEST_CA_CRT_RSA_SHA1_DER tests/data_files/test-ca-sha1.crt.der */ +#define TEST_CA_CRT_RSA_SHA1_DER { \ + 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ + 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ + 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ + 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ + 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ + 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ + 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ + 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ + 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ + 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ + 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ + 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ + 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ + 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ + 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ + 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ + 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ + 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ + 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ + 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ + 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ + 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ + 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ + 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ + 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ + 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ + 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ + 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ + 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ + 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x01, 0x00, 0x01, 0x13, 0x73, 0x84, 0x3d, 0xf1, 0x1d, \ + 0xfd, 0xb7, 0x09, 0x5b, 0x96, 0x5d, 0x53, 0x7f, 0xd5, 0x80, 0xf3, 0x52, \ + 0xe2, 0xd3, 0x33, 0x87, 0xc8, 0x27, 0x24, 0xff, 0xd5, 0xd8, 0x57, 0x2f, \ + 0x16, 0xd1, 0xb2, 0x94, 0xca, 0x50, 0xab, 0xa6, 0x27, 0x10, 0x16, 0x08, \ + 0xc8, 0x11, 0xc0, 0x2f, 0x80, 0xd1, 0xbe, 0x53, 0x18, 0xe6, 0xb9, 0xd7, \ + 0x18, 0x1a, 0x77, 0x38, 0x34, 0x7c, 0x32, 0x9a, 0x87, 0x0b, 0xa0, 0x2a, \ + 0xb9, 0x14, 0xc2, 0x2f, 0x38, 0xd2, 0xe7, 0xb8, 0x98, 0x7d, 0xff, 0xff, \ + 0xe1, 0x01, 0x50, 0xa9, 0x6f, 0x67, 0xf7, 0x6c, 0xdc, 0xb6, 0xca, 0x6f, \ + 0x73, 0x39, 0x1a, 0x3c, 0xa8, 0x23, 0xaa, 0x8d, 0x4d, 0xa3, 0x75, 0x2a, \ + 0xd1, 0x76, 0xb3, 0xd7, 0x4a, 0xdc, 0xc7, 0x24, 0xd4, 0x3e, 0xb7, 0xf9, \ + 0xc0, 0xd5, 0x51, 0x67, 0x65, 0x74, 0x2a, 0xf9, 0x65, 0xbc, 0x00, 0x15, \ + 0x4b, 0x36, 0xc8, 0xe2, 0x6a, 0x5d, 0x51, 0x7c, 0xed, 0x8e, 0x14, 0x93, \ + 0x4b, 0x90, 0x36, 0x05, 0xe5, 0x90, 0x00, 0x03, 0xab, 0xd3, 0x3a, 0xb5, \ + 0x17, 0xb4, 0xd2, 0x45, 0x52, 0x69, 0x26, 0xce, 0xe3, 0x98, 0x1d, 0x9a, \ + 0x8b, 0xf8, 0xa0, 0x92, 0x1d, 0x48, 0x02, 0x37, 0x2e, 0xc1, 0x5e, 0x95, \ + 0xc2, 0x53, 0xfe, 0xb1, 0xbc, 0x34, 0x82, 0x34, 0x34, 0x36, 0x91, 0x8c, \ + 0x88, 0x7a, 0x67, 0x97, 0x34, 0x40, 0x8b, 0xfb, 0x48, 0x6e, 0xd3, 0xaf, \ + 0x30, 0x81, 0x8e, 0x05, 0x4d, 0x93, 0x21, 0xf6, 0xb1, 0xff, 0x98, 0xea, \ + 0xd5, 0xa8, 0x14, 0xc7, 0x96, 0x8f, 0x99, 0x3e, 0x53, 0x58, 0x08, 0x89, \ + 0x3c, 0xe3, 0x8f, 0xea, 0x5e, 0x71, 0x5e, 0x70, 0xf0, 0xc5, 0xe6, 0x12, \ + 0x35, 0x6a, 0xa2, 0x5f, 0xd1, 0xb2, 0xba, 0xc0, 0x59, 0x8d, 0xec, 0xda, \ + 0x09, 0xa1, 0xda, 0x6e, 0x30, 0xcb, 0x53, 0x4a, 0x90 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/test-ca.key */ +/* BEGIN FILE string macro TEST_CA_KEY_RSA_PEM tests/data_files/test-ca.key */ +#define TEST_CA_KEY_RSA_PEM \ + "-----BEGIN RSA PRIVATE KEY-----\r\n" \ + "Proc-Type: 4,ENCRYPTED\r\n" \ + "DEK-Info: DES-EDE3-CBC,A8A95B05D5B7206B\r\n" \ + "\r\n" \ + "9Qd9GeArejl1GDVh2lLV1bHt0cPtfbh5h/5zVpAVaFpqtSPMrElp50Rntn9et+JA\r\n" \ + "7VOyboR+Iy2t/HU4WvA687k3Bppe9GwKHjHhtl//8xFKwZr3Xb5yO5JUP8AUctQq\r\n" \ + "Nb8CLlZyuUC+52REAAthdWgsX+7dJO4yabzUcQ22Tp9JSD0hiL43BlkWYUNK3dAo\r\n" \ + "PZlmiptjnzVTjg1MxsBSydZinWOLBV8/JQgxSPo2yD4uEfig28qbvQ2wNIn0pnAb\r\n" \ + "GxnSAOazkongEGfvcjIIs+LZN9gXFhxcOh6kc4Q/c99B7QWETwLLkYgZ+z1a9VY9\r\n" \ + "gEU7CwCxYCD+h9hY6FPmsK0/lC4O7aeRKpYq00rPPxs6i7phiexg6ax6yTMmArQq\r\n" \ + "QmK3TAsJm8V/J5AWpLEV6jAFgRGymGGHnof0DXzVWZidrcZJWTNuGEX90nB3ee2w\r\n" \ + "PXJEFWKoD3K3aFcSLdHYr3mLGxP7H9ThQai9VsycxZKS5kwvBKQ//YMrmFfwPk8x\r\n" \ + "vTeY4KZMaUrveEel5tWZC94RSMKgxR6cyE1nBXyTQnDOGbfpNNgBKxyKbINWoOJU\r\n" \ + "WJZAwlsQn+QzCDwpri7+sV1mS3gBE6UY7aQmnmiiaC2V3Hbphxct/en5QsfDOt1X\r\n" \ + "JczSfpRWLlbPznZg8OQh/VgCMA58N5DjOzTIK7sJJ5r+94ZBTCpgAMbF588f0NTR\r\n" \ + "KCe4yrxGJR7X02M4nvD4IwOlpsQ8xQxZtOSgXv4LkxvdU9XJJKWZ/XNKJeWztxSe\r\n" \ + "Z1vdTc2YfsDBA2SEv33vxHx2g1vqtw8SjDRT2RaQSS0QuSaMJimdOX6mTOCBKk1J\r\n" \ + "9Q5mXTrER+/LnK0jEmXsBXWA5bqqVZIyahXSx4VYZ7l7w/PHiUDtDgyRhMMKi4n2\r\n" \ + "iQvQcWSQTjrpnlJbca1/DkpRt3YwrvJwdqb8asZU2VrNETh5x0QVefDRLFiVpif/\r\n" \ + "tUaeAe/P1F8OkS7OIZDs1SUbv/sD2vMbhNkUoCms3/PvNtdnvgL4F0zhaDpKCmlT\r\n" \ + "P8vx49E7v5CyRNmED9zZg4o3wmMqrQO93PtTug3Eu9oVx1zPQM1NVMyBa2+f29DL\r\n" \ + "1nuTCeXdo9+ni45xx+jAI4DCwrRdhJ9uzZyC6962H37H6D+5naNvClFR1s6li1Gb\r\n" \ + "nqPoiy/OBsEx9CaDGcqQBp5Wme/3XW+6z1ISOx+igwNTVCT14mHdBMbya0eIKft5\r\n" \ + "X+GnwtgEMyCYyyWuUct8g4RzErcY9+yW9Om5Hzpx4zOuW4NPZgPDTgK+t2RSL/Yq\r\n" \ + "rE1njrgeGYcVeG3f+OftH4s6fPbq7t1A5ZgUscbLMBqr9tK+OqygR4EgKBPsH6Cz\r\n" \ + "L6zlv/2RV0qAHvVuDJcIDIgwY5rJtINEm32rhOeFNJwZS5MNIC1czXZx5//ugX7l\r\n" \ + "I4sy5nbVhwSjtAk8Xg5dZbdTZ6mIrb7xqH+fdakZor1khG7bC2uIwibD3cSl2XkR\r\n" \ + "wN48lslbHnqqagr6Xm1nNOSVl8C/6kbJEsMpLhAezfRtGwvOucoaE+WbeUNolGde\r\n" \ + "P/eQiddSf0brnpiLJRh7qZrl9XuqYdpUqnoEdMAfotDOID8OtV7gt8a48ad8VPW2\r\n" \ + "-----END RSA PRIVATE KEY-----\r\n" +/* END FILE */ + +#define TEST_CA_PWD_RSA_PEM "PolarSSLTest" + +/* This was generated from test-ca.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CA_KEY_RSA_DER tests/data_files/test-ca.key.der */ +#define TEST_CA_KEY_RSA_DER { \ + 0x30, 0x82, 0x04, 0xa4, 0x02, 0x01, 0x00, 0x02, 0x82, 0x01, 0x01, 0x00, \ + 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, 0x86, 0xde, \ + 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, 0x99, 0xd4, \ + 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, 0x9b, 0xc5, \ + 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, 0xc0, 0x8d, \ + 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, 0x93, 0xe8, \ + 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, 0xe7, 0x40, \ + 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, 0xf9, 0x3e, \ + 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, 0x29, 0x00, \ + 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, 0xbd, 0x83, \ + 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, 0x60, 0xc3, \ + 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, 0x32, 0xbe, \ + 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, 0xfb, 0xf5, \ + 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, 0xee, 0xe2, \ + 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, 0x47, 0xb1, \ + 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, 0xf1, 0x79, \ + 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, 0x6f, 0x27, \ + 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, 0xa1, 0x30, \ + 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, 0x28, 0xd1, \ + 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, 0x09, 0xea, \ + 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, 0xc9, 0xab, \ + 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, 0x9e, 0x99, \ + 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0x02, 0x82, 0x01, \ + 0x00, 0x3f, 0xf7, 0x07, 0xd3, 0x34, 0x6f, 0xdb, 0xc9, 0x37, 0xb7, 0x84, \ + 0xdc, 0x37, 0x45, 0xe1, 0x63, 0xad, 0xb8, 0xb6, 0x75, 0xb1, 0xc7, 0x35, \ + 0xb4, 0x77, 0x2a, 0x5b, 0x77, 0xf9, 0x7e, 0xe0, 0xc1, 0xa3, 0xd1, 0xb7, \ + 0xcb, 0xa9, 0x5a, 0xc1, 0x87, 0xda, 0x5a, 0xfa, 0x17, 0xe4, 0xd5, 0x38, \ + 0x03, 0xde, 0x68, 0x98, 0x81, 0xec, 0xb5, 0xf2, 0x2a, 0x8d, 0xe9, 0x2c, \ + 0xf3, 0xa6, 0xe5, 0x32, 0x17, 0x7f, 0x33, 0x81, 0xe8, 0x38, 0x72, 0xd5, \ + 0x9c, 0xfa, 0x4e, 0xfb, 0x26, 0xf5, 0x15, 0x0b, 0xaf, 0x84, 0x66, 0xab, \ + 0x02, 0xe0, 0x18, 0xd5, 0x91, 0x7c, 0xd6, 0x8f, 0xc9, 0x4b, 0x76, 0x08, \ + 0x2b, 0x1d, 0x81, 0x68, 0x30, 0xe1, 0xfa, 0x70, 0x6c, 0x13, 0x4e, 0x10, \ + 0x03, 0x35, 0x3e, 0xc5, 0xca, 0x58, 0x20, 0x8a, 0x21, 0x18, 0x38, 0xa0, \ + 0x0f, 0xed, 0xc4, 0xbb, 0x45, 0x6f, 0xf5, 0x84, 0x5b, 0xb0, 0xcf, 0x4e, \ + 0x9d, 0x58, 0x13, 0x6b, 0x35, 0x35, 0x69, 0xa1, 0xd2, 0xc4, 0xf2, 0xc1, \ + 0x48, 0x04, 0x20, 0x51, 0xb9, 0x6b, 0xa4, 0x5d, 0xa5, 0x4b, 0x84, 0x88, \ + 0x43, 0x48, 0x99, 0x2c, 0xbb, 0xa4, 0x97, 0xd6, 0xd6, 0x18, 0xf6, 0xec, \ + 0x5c, 0xd1, 0x31, 0x49, 0xc9, 0xf2, 0x8f, 0x0b, 0x4d, 0xef, 0x09, 0x02, \ + 0xfe, 0x7d, 0xfd, 0xbb, 0xaf, 0x2b, 0x83, 0x94, 0x22, 0xc4, 0xa7, 0x3e, \ + 0x66, 0xf5, 0xe0, 0x57, 0xdc, 0xf2, 0xed, 0x2c, 0x3e, 0x81, 0x74, 0x76, \ + 0x1e, 0x96, 0x6f, 0x74, 0x1e, 0x32, 0x0e, 0x14, 0x31, 0xd0, 0x74, 0xf0, \ + 0xf4, 0x07, 0xbd, 0xc3, 0xd1, 0x22, 0xc2, 0xa8, 0x95, 0x92, 0x06, 0x7f, \ + 0x43, 0x02, 0x91, 0xbc, 0xdd, 0x23, 0x01, 0x89, 0x94, 0x20, 0x44, 0x64, \ + 0xf5, 0x1d, 0x67, 0xd2, 0x8f, 0xe8, 0x69, 0xa5, 0x29, 0x25, 0xe6, 0x50, \ + 0x9c, 0xe3, 0xe9, 0xcb, 0x75, 0x02, 0x81, 0x81, 0x00, 0xe2, 0x29, 0x3e, \ + 0xaa, 0x6b, 0xd5, 0x59, 0x1e, 0x9c, 0xe6, 0x47, 0xd5, 0xb6, 0xd7, 0xe3, \ + 0xf1, 0x8e, 0x9e, 0xe9, 0x83, 0x5f, 0x10, 0x9f, 0x63, 0xec, 0x04, 0x44, \ + 0xcc, 0x3f, 0xf8, 0xd9, 0x3a, 0x17, 0xe0, 0x4f, 0xfe, 0xd8, 0x4d, 0xcd, \ + 0x46, 0x54, 0x74, 0xbf, 0x0a, 0xc4, 0x67, 0x9c, 0xa7, 0xd8, 0x89, 0x65, \ + 0x4c, 0xfd, 0x58, 0x2a, 0x47, 0x0f, 0xf4, 0x37, 0xb6, 0x55, 0xb0, 0x1d, \ + 0xed, 0xa7, 0x39, 0xfc, 0x4f, 0xa3, 0xc4, 0x75, 0x3a, 0xa3, 0x98, 0xa7, \ + 0x45, 0xf5, 0x66, 0xcb, 0x7c, 0x65, 0xfb, 0x80, 0x23, 0xe6, 0xff, 0xfd, \ + 0x99, 0x1f, 0x8e, 0x6b, 0xff, 0x5e, 0x93, 0x66, 0xdf, 0x6c, 0x6f, 0xc3, \ + 0xf6, 0x38, 0x2e, 0xff, 0x69, 0xb5, 0xac, 0xae, 0xbb, 0xc6, 0x71, 0x16, \ + 0x6b, 0xd0, 0xf8, 0x22, 0xd9, 0xf8, 0xa2, 0x72, 0x20, 0xd2, 0xe2, 0x3a, \ + 0x70, 0x4b, 0xde, 0xab, 0x2f, 0x02, 0x81, 0x81, 0x00, 0xda, 0x51, 0x9b, \ + 0xb8, 0xb2, 0x2a, 0x14, 0x75, 0x58, 0x40, 0x8d, 0x27, 0x70, 0xfa, 0x31, \ + 0x48, 0xb0, 0x20, 0x21, 0x34, 0xfa, 0x4c, 0x57, 0xa8, 0x11, 0x88, 0xf3, \ + 0xa7, 0xae, 0x21, 0xe9, 0xb6, 0x2b, 0xd1, 0xcd, 0xa7, 0xf8, 0xd8, 0x0c, \ + 0x8a, 0x76, 0x22, 0x35, 0x44, 0xce, 0x3f, 0x25, 0x29, 0x83, 0x7d, 0x79, \ + 0xa7, 0x31, 0xd6, 0xec, 0xb2, 0xbf, 0xda, 0x34, 0xb6, 0xf6, 0xb2, 0x3b, \ + 0xf3, 0x78, 0x5a, 0x04, 0x83, 0x33, 0x3e, 0xa2, 0xe2, 0x81, 0x82, 0x13, \ + 0xd4, 0x35, 0x17, 0x63, 0x9b, 0x9e, 0xc4, 0x8d, 0x91, 0x4c, 0x03, 0x77, \ + 0xc7, 0x71, 0x5b, 0xee, 0x83, 0x6d, 0xd5, 0x78, 0x88, 0xf6, 0x2c, 0x79, \ + 0xc2, 0x4a, 0xb4, 0x79, 0x90, 0x70, 0xbf, 0xdf, 0x34, 0x56, 0x96, 0x71, \ + 0xe3, 0x0e, 0x68, 0x91, 0xbc, 0xea, 0xcb, 0x33, 0xc0, 0xbe, 0x45, 0xd7, \ + 0xfc, 0x30, 0xfd, 0x01, 0x3b, 0x02, 0x81, 0x81, 0x00, 0xd2, 0x9f, 0x2a, \ + 0xb7, 0x38, 0x19, 0xc7, 0x17, 0x95, 0x73, 0x78, 0xae, 0xf5, 0xcb, 0x75, \ + 0x83, 0x7f, 0x19, 0x4b, 0xcb, 0x86, 0xfb, 0x4a, 0x15, 0x9a, 0xb6, 0x17, \ + 0x04, 0x49, 0x07, 0x8d, 0xf6, 0x66, 0x4a, 0x06, 0xf6, 0x05, 0xa7, 0xdf, \ + 0x66, 0x82, 0x3c, 0xff, 0xb6, 0x1d, 0x57, 0x89, 0x33, 0x5f, 0x9c, 0x05, \ + 0x75, 0x7f, 0xf3, 0x5d, 0xdc, 0x34, 0x65, 0x72, 0x85, 0x22, 0xa4, 0x14, \ + 0x1b, 0x41, 0xc3, 0xe4, 0xd0, 0x9e, 0x69, 0xd5, 0xeb, 0x38, 0x74, 0x70, \ + 0x43, 0xdc, 0xd9, 0x50, 0xe4, 0x97, 0x6d, 0x73, 0xd6, 0xfb, 0xc8, 0xa7, \ + 0xfa, 0xb4, 0xc2, 0xc4, 0x9d, 0x5d, 0x0c, 0xd5, 0x9f, 0x79, 0xb3, 0x54, \ + 0xc2, 0xb7, 0x6c, 0x3d, 0x7d, 0xcb, 0x2d, 0xf8, 0xc4, 0xf3, 0x78, 0x5a, \ + 0x33, 0x2a, 0xb8, 0x0c, 0x6d, 0x06, 0xfa, 0xf2, 0x62, 0xd3, 0x42, 0xd0, \ + 0xbd, 0xc8, 0x4a, 0xa5, 0x0d, 0x02, 0x81, 0x81, 0x00, 0xd4, 0xa9, 0x90, \ + 0x15, 0xde, 0xbf, 0x2c, 0xc4, 0x8d, 0x9d, 0xfb, 0xa1, 0xc2, 0xe4, 0x83, \ + 0xe3, 0x79, 0x65, 0x22, 0xd3, 0xb7, 0x49, 0x6c, 0x4d, 0x94, 0x1f, 0x22, \ + 0xb1, 0x60, 0xe7, 0x3a, 0x00, 0xb1, 0x38, 0xa2, 0xab, 0x0f, 0xb4, 0x6c, \ + 0xaa, 0xe7, 0x9e, 0x34, 0xe3, 0x7c, 0x40, 0x78, 0x53, 0xb2, 0xf9, 0x23, \ + 0xea, 0xa0, 0x9a, 0xea, 0x60, 0xc8, 0x8f, 0xa6, 0xaf, 0xdf, 0x29, 0x09, \ + 0x4b, 0x06, 0x1e, 0x31, 0xad, 0x17, 0xda, 0xd8, 0xd1, 0xe9, 0x33, 0xab, \ + 0x5b, 0x18, 0x08, 0x5b, 0x87, 0xf8, 0xa5, 0x1f, 0xfd, 0xbb, 0xdc, 0xd8, \ + 0xed, 0x97, 0x57, 0xe4, 0xc3, 0x73, 0xd6, 0xf0, 0x9e, 0x01, 0xa6, 0x9b, \ + 0x48, 0x8e, 0x7a, 0xb4, 0xbb, 0xe5, 0x88, 0x91, 0xc5, 0x2a, 0xdf, 0x4b, \ + 0xba, 0xd0, 0x8b, 0x3e, 0x03, 0x97, 0x77, 0x2f, 0x47, 0x7e, 0x51, 0x0c, \ + 0xae, 0x65, 0x8d, 0xde, 0x87, 0x02, 0x81, 0x80, 0x20, 0x24, 0x0f, 0xd2, \ + 0xaf, 0xc2, 0x28, 0x3b, 0x97, 0x20, 0xb2, 0x92, 0x49, 0xeb, 0x09, 0x68, \ + 0x40, 0xb2, 0xbe, 0xd1, 0xc3, 0x83, 0x94, 0x34, 0x38, 0xd6, 0xc9, 0xec, \ + 0x34, 0x09, 0xf9, 0x41, 0x6d, 0x5c, 0x42, 0x94, 0xf7, 0x04, 0xfc, 0x32, \ + 0x39, 0x69, 0xbc, 0x1c, 0xfb, 0x3e, 0x61, 0x98, 0xc0, 0x80, 0xd8, 0x36, \ + 0x47, 0xc3, 0x6d, 0xc2, 0x2e, 0xe7, 0x81, 0x2a, 0x17, 0x34, 0x64, 0x30, \ + 0x4e, 0x96, 0xbb, 0x26, 0x16, 0xb9, 0x41, 0x36, 0xfe, 0x8a, 0xd6, 0x53, \ + 0x7c, 0xaa, 0xec, 0x39, 0x42, 0x50, 0xef, 0xe3, 0xb3, 0x01, 0x28, 0x32, \ + 0xca, 0x6d, 0xf5, 0x9a, 0x1e, 0x9f, 0x37, 0xbe, 0xfe, 0x38, 0x20, 0x22, \ + 0x91, 0x8c, 0xcd, 0x95, 0x02, 0xf2, 0x4d, 0x6f, 0x1a, 0xb4, 0x43, 0xf0, \ + 0x19, 0xdf, 0x65, 0xc0, 0x92, 0xe7, 0x9d, 0x2f, 0x09, 0xe7, 0xec, 0x69, \ + 0xa8, 0xc2, 0x8f, 0x0d \ +} +/* END FILE */ + +/* + * Test server Certificates + * + * Test server certificates are defined for each choice + * of the following parameters: + * - PEM or DER encoding + * - SHA-1 or SHA-256 hash + * - RSA or EC key + * + * Things to add: + * - multiple EC curve types + */ + +/* This is taken from tests/data_files/server5.crt. */ +/* BEGIN FILE string macro TEST_SRV_CRT_EC_PEM tests/data_files/server5.crt */ +#define TEST_SRV_CRT_EC_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIICHzCCAaWgAwIBAgIBCTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\r\n" \ + "MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UEChMIUG9sYXJTU0wxEjAQBgNVBAMTCWxvY2FsaG9zdDBZMBMGByqGSM49AgEG\r\n" \ + "CCqGSM49AwEHA0IABDfMVtl2CR5acj7HWS3/IG7ufPkGkXTQrRS192giWWKSTuUA\r\n" \ + "2CMR/+ov0jRdXRa9iojCa3cNVc2KKg76Aci07f+jgZ0wgZowCQYDVR0TBAIwADAd\r\n" \ + "BgNVHQ4EFgQUUGGlj9QH2deCAQzlZX+MY0anE74wbgYDVR0jBGcwZYAUnW0gJEkB\r\n" \ + "PyvLeLUZvH4kydv7NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xh\r\n" \ + "clNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAoG\r\n" \ + "CCqGSM49BAMCA2gAMGUCMQCaLFzXptui5WQN8LlO3ddh1hMxx6tzgLvT03MTVK2S\r\n" \ + "C12r0Lz3ri/moSEpNZWqPjkCMCE2f53GXcYLqyfyJR078c/xNSUU5+Xxl7VZ414V\r\n" \ + "fGa5kHvHARBPc8YAIVIqDvHH1Q==\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is generated from tests/data_files/server5.crt.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_SRV_CRT_EC_DER tests/data_files/server5.crt.der */ +#define TEST_SRV_CRT_EC_DER { \ + 0x30, 0x82, 0x02, 0x1f, 0x30, 0x82, 0x01, 0xa5, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x09, 0x30, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, \ + 0x3d, 0x04, 0x03, 0x02, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ + 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x33, 0x30, 0x39, 0x32, 0x34, 0x31, 0x35, 0x35, 0x32, 0x30, 0x34, \ + 0x5a, 0x17, 0x0d, 0x32, 0x33, 0x30, 0x39, 0x32, 0x32, 0x31, 0x35, 0x35, \ + 0x32, 0x30, 0x34, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x59, \ + 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, \ + 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07, 0x03, 0x42, 0x00, \ + 0x04, 0x37, 0xcc, 0x56, 0xd9, 0x76, 0x09, 0x1e, 0x5a, 0x72, 0x3e, 0xc7, \ + 0x59, 0x2d, 0xff, 0x20, 0x6e, 0xee, 0x7c, 0xf9, 0x06, 0x91, 0x74, 0xd0, \ + 0xad, 0x14, 0xb5, 0xf7, 0x68, 0x22, 0x59, 0x62, 0x92, 0x4e, 0xe5, 0x00, \ + 0xd8, 0x23, 0x11, 0xff, 0xea, 0x2f, 0xd2, 0x34, 0x5d, 0x5d, 0x16, 0xbd, \ + 0x8a, 0x88, 0xc2, 0x6b, 0x77, 0x0d, 0x55, 0xcd, 0x8a, 0x2a, 0x0e, 0xfa, \ + 0x01, 0xc8, 0xb4, 0xed, 0xff, 0xa3, 0x81, 0x9d, 0x30, 0x81, 0x9a, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, \ + 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x50, 0x61, 0xa5, \ + 0x8f, 0xd4, 0x07, 0xd9, 0xd7, 0x82, 0x01, 0x0c, 0xe5, 0x65, 0x7f, 0x8c, \ + 0x63, 0x46, 0xa7, 0x13, 0xbe, 0x30, 0x6e, 0x06, 0x03, 0x55, 0x1d, 0x23, \ + 0x04, 0x67, 0x30, 0x65, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, 0x01, \ + 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, 0xfb, \ + 0x36, 0x7c, 0xa1, 0x42, 0xa4, 0x40, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, \ + 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, \ + 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, \ + 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, \ + 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, \ + 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x82, 0x09, \ + 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, 0x30, 0x0a, 0x06, \ + 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, 0x03, 0x68, 0x00, \ + 0x30, 0x65, 0x02, 0x31, 0x00, 0x9a, 0x2c, 0x5c, 0xd7, 0xa6, 0xdb, 0xa2, \ + 0xe5, 0x64, 0x0d, 0xf0, 0xb9, 0x4e, 0xdd, 0xd7, 0x61, 0xd6, 0x13, 0x31, \ + 0xc7, 0xab, 0x73, 0x80, 0xbb, 0xd3, 0xd3, 0x73, 0x13, 0x54, 0xad, 0x92, \ + 0x0b, 0x5d, 0xab, 0xd0, 0xbc, 0xf7, 0xae, 0x2f, 0xe6, 0xa1, 0x21, 0x29, \ + 0x35, 0x95, 0xaa, 0x3e, 0x39, 0x02, 0x30, 0x21, 0x36, 0x7f, 0x9d, 0xc6, \ + 0x5d, 0xc6, 0x0b, 0xab, 0x27, 0xf2, 0x25, 0x1d, 0x3b, 0xf1, 0xcf, 0xf1, \ + 0x35, 0x25, 0x14, 0xe7, 0xe5, 0xf1, 0x97, 0xb5, 0x59, 0xe3, 0x5e, 0x15, \ + 0x7c, 0x66, 0xb9, 0x90, 0x7b, 0xc7, 0x01, 0x10, 0x4f, 0x73, 0xc6, 0x00, \ + 0x21, 0x52, 0x2a, 0x0e, 0xf1, 0xc7, 0xd5 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/server5.key. */ +/* BEGIN FILE string macro TEST_SRV_KEY_EC_PEM tests/data_files/server5.key */ +#define TEST_SRV_KEY_EC_PEM \ + "-----BEGIN EC PRIVATE KEY-----\r\n" \ + "MHcCAQEEIPEqEyB2AnCoPL/9U/YDHvdqXYbIogTywwyp6/UfDw6noAoGCCqGSM49\r\n" \ + "AwEHoUQDQgAEN8xW2XYJHlpyPsdZLf8gbu58+QaRdNCtFLX3aCJZYpJO5QDYIxH/\r\n" \ + "6i/SNF1dFr2KiMJrdw1VzYoqDvoByLTt/w==\r\n" \ + "-----END EC PRIVATE KEY-----\r\n" +/* END FILE */ + +/* This is generated from tests/data_files/server5.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_SRV_KEY_EC_DER tests/data_files/server5.key.der */ +#define TEST_SRV_KEY_EC_DER { \ + 0x30, 0x77, 0x02, 0x01, 0x01, 0x04, 0x20, 0xf1, 0x2a, 0x13, 0x20, 0x76, \ + 0x02, 0x70, 0xa8, 0x3c, 0xbf, 0xfd, 0x53, 0xf6, 0x03, 0x1e, 0xf7, 0x6a, \ + 0x5d, 0x86, 0xc8, 0xa2, 0x04, 0xf2, 0xc3, 0x0c, 0xa9, 0xeb, 0xf5, 0x1f, \ + 0x0f, 0x0e, 0xa7, 0xa0, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, \ + 0x03, 0x01, 0x07, 0xa1, 0x44, 0x03, 0x42, 0x00, 0x04, 0x37, 0xcc, 0x56, \ + 0xd9, 0x76, 0x09, 0x1e, 0x5a, 0x72, 0x3e, 0xc7, 0x59, 0x2d, 0xff, 0x20, \ + 0x6e, 0xee, 0x7c, 0xf9, 0x06, 0x91, 0x74, 0xd0, 0xad, 0x14, 0xb5, 0xf7, \ + 0x68, 0x22, 0x59, 0x62, 0x92, 0x4e, 0xe5, 0x00, 0xd8, 0x23, 0x11, 0xff, \ + 0xea, 0x2f, 0xd2, 0x34, 0x5d, 0x5d, 0x16, 0xbd, 0x8a, 0x88, 0xc2, 0x6b, \ + 0x77, 0x0d, 0x55, 0xcd, 0x8a, 0x2a, 0x0e, 0xfa, 0x01, 0xc8, 0xb4, 0xed, \ + 0xff \ +} +/* END FILE */ + +/* This is taken from tests/data_files/server2-sha256.crt. */ +/* BEGIN FILE string macro TEST_SRV_CRT_RSA_SHA256_PEM tests/data_files/server2-sha256.crt */ +#define TEST_SRV_CRT_RSA_SHA256_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIIDNzCCAh+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ + "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UECgwIUG9sYXJTU0wxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN\r\n" \ + "AQEBBQADggEPADCCAQoCggEBAMFNo93nzR3RBNdJcriZrA545Do8Ss86ExbQWuTN\r\n" \ + "owCIp+4ea5anUrSQ7y1yej4kmvy2NKwk9XfgJmSMnLAofaHa6ozmyRyWvP7BBFKz\r\n" \ + "NtSj+uGxdtiQwWG0ZlI2oiZTqqt0Xgd9GYLbKtgfoNkNHC1JZvdbJXNG6AuKT2kM\r\n" \ + "tQCQ4dqCEGZ9rlQri2V5kaHiYcPNQEkI7mgM8YuG0ka/0LiqEQMef1aoGh5EGA8P\r\n" \ + "hYvai0Re4hjGYi/HZo36Xdh98yeJKQHFkA4/J/EwyEoO79bex8cna8cFPXrEAjya\r\n" \ + "HT4P6DSYW8tzS1KW2BGiLICIaTla0w+w3lkvEcf36hIBMJcCAwEAAaNNMEswCQYD\r\n" \ + "VR0TBAIwADAdBgNVHQ4EFgQUpQXoZLjc32APUBJNYKhkr02LQ5MwHwYDVR0jBBgw\r\n" \ + "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQELBQADggEBAGGEshT5\r\n" \ + "kvnRmLVScVeUEdwIrvW7ezbGbUvJ8VxeJ79/HSjlLiGbMc4uUathwtzEdi9R/4C5\r\n" \ + "DXBNeEPTkbB+fhG1W06iHYj/Dp8+aaG7fuDxKVKHVZSqBnmQLn73ymyclZNHii5A\r\n" \ + "3nTS8WUaHAzxN/rajOtoM7aH1P9tULpHrl+7HOeLMpxUnwI12ZqZaLIzxbcdJVcr\r\n" \ + "ra2F00aXCGkYVLvyvbZIq7LC+yVysej5gCeQYD7VFOEks0jhFjrS06gP0/XnWv6v\r\n" \ + "eBoPez9d+CCjkrhseiWzXOiriIMICX48EloO/DrsMRAtvlwq7EDz4QhILz6ffndm\r\n" \ + "e4K1cVANRPN2o9Y=\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is taken from tests/data_files/server2-sha256.crt.der. */ +/* BEGIN FILE binary macro TEST_SRV_CRT_RSA_SHA256_DER tests/data_files/server2-sha256.crt.der */ +#define TEST_SRV_CRT_RSA_SHA256_DER { \ + 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ + 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ + 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ + 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ + 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ + 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ + 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ + 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ + 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ + 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ + 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ + 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ + 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ + 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ + 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ + 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ + 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ + 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ + 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ + 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ + 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ + 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ + 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ + 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ + 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ + 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ + 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ + 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ + 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, \ + 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x61, 0x84, 0xb2, 0x14, 0xf9, \ + 0x92, 0xf9, 0xd1, 0x98, 0xb5, 0x52, 0x71, 0x57, 0x94, 0x11, 0xdc, 0x08, \ + 0xae, 0xf5, 0xbb, 0x7b, 0x36, 0xc6, 0x6d, 0x4b, 0xc9, 0xf1, 0x5c, 0x5e, \ + 0x27, 0xbf, 0x7f, 0x1d, 0x28, 0xe5, 0x2e, 0x21, 0x9b, 0x31, 0xce, 0x2e, \ + 0x51, 0xab, 0x61, 0xc2, 0xdc, 0xc4, 0x76, 0x2f, 0x51, 0xff, 0x80, 0xb9, \ + 0x0d, 0x70, 0x4d, 0x78, 0x43, 0xd3, 0x91, 0xb0, 0x7e, 0x7e, 0x11, 0xb5, \ + 0x5b, 0x4e, 0xa2, 0x1d, 0x88, 0xff, 0x0e, 0x9f, 0x3e, 0x69, 0xa1, 0xbb, \ + 0x7e, 0xe0, 0xf1, 0x29, 0x52, 0x87, 0x55, 0x94, 0xaa, 0x06, 0x79, 0x90, \ + 0x2e, 0x7e, 0xf7, 0xca, 0x6c, 0x9c, 0x95, 0x93, 0x47, 0x8a, 0x2e, 0x40, \ + 0xde, 0x74, 0xd2, 0xf1, 0x65, 0x1a, 0x1c, 0x0c, 0xf1, 0x37, 0xfa, 0xda, \ + 0x8c, 0xeb, 0x68, 0x33, 0xb6, 0x87, 0xd4, 0xff, 0x6d, 0x50, 0xba, 0x47, \ + 0xae, 0x5f, 0xbb, 0x1c, 0xe7, 0x8b, 0x32, 0x9c, 0x54, 0x9f, 0x02, 0x35, \ + 0xd9, 0x9a, 0x99, 0x68, 0xb2, 0x33, 0xc5, 0xb7, 0x1d, 0x25, 0x57, 0x2b, \ + 0xad, 0xad, 0x85, 0xd3, 0x46, 0x97, 0x08, 0x69, 0x18, 0x54, 0xbb, 0xf2, \ + 0xbd, 0xb6, 0x48, 0xab, 0xb2, 0xc2, 0xfb, 0x25, 0x72, 0xb1, 0xe8, 0xf9, \ + 0x80, 0x27, 0x90, 0x60, 0x3e, 0xd5, 0x14, 0xe1, 0x24, 0xb3, 0x48, 0xe1, \ + 0x16, 0x3a, 0xd2, 0xd3, 0xa8, 0x0f, 0xd3, 0xf5, 0xe7, 0x5a, 0xfe, 0xaf, \ + 0x78, 0x1a, 0x0f, 0x7b, 0x3f, 0x5d, 0xf8, 0x20, 0xa3, 0x92, 0xb8, 0x6c, \ + 0x7a, 0x25, 0xb3, 0x5c, 0xe8, 0xab, 0x88, 0x83, 0x08, 0x09, 0x7e, 0x3c, \ + 0x12, 0x5a, 0x0e, 0xfc, 0x3a, 0xec, 0x31, 0x10, 0x2d, 0xbe, 0x5c, 0x2a, \ + 0xec, 0x40, 0xf3, 0xe1, 0x08, 0x48, 0x2f, 0x3e, 0x9f, 0x7e, 0x77, 0x66, \ + 0x7b, 0x82, 0xb5, 0x71, 0x50, 0x0d, 0x44, 0xf3, 0x76, 0xa3, 0xd6 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/server2.crt. */ +/* BEGIN FILE string macro TEST_SRV_CRT_RSA_SHA1_PEM tests/data_files/server2.crt */ +#define TEST_SRV_CRT_RSA_SHA1_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIIDNzCCAh+gAwIBAgIBAjANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ + "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UECgwIUG9sYXJTU0wxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN\r\n" \ + "AQEBBQADggEPADCCAQoCggEBAMFNo93nzR3RBNdJcriZrA545Do8Ss86ExbQWuTN\r\n" \ + "owCIp+4ea5anUrSQ7y1yej4kmvy2NKwk9XfgJmSMnLAofaHa6ozmyRyWvP7BBFKz\r\n" \ + "NtSj+uGxdtiQwWG0ZlI2oiZTqqt0Xgd9GYLbKtgfoNkNHC1JZvdbJXNG6AuKT2kM\r\n" \ + "tQCQ4dqCEGZ9rlQri2V5kaHiYcPNQEkI7mgM8YuG0ka/0LiqEQMef1aoGh5EGA8P\r\n" \ + "hYvai0Re4hjGYi/HZo36Xdh98yeJKQHFkA4/J/EwyEoO79bex8cna8cFPXrEAjya\r\n" \ + "HT4P6DSYW8tzS1KW2BGiLICIaTla0w+w3lkvEcf36hIBMJcCAwEAAaNNMEswCQYD\r\n" \ + "VR0TBAIwADAdBgNVHQ4EFgQUpQXoZLjc32APUBJNYKhkr02LQ5MwHwYDVR0jBBgw\r\n" \ + "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQEFBQADggEBAAFzC0rF\r\n" \ + "y6De8WMcdgQrEw3AhBHFjzqnxZw1ene4IBSC7lTw8rBSy3jOWQdPUWn+0y/pCeeF\r\n" \ + "kti6sevFdl1hLemGtd4q+T9TKEKGg3ND4ARfB5AUZZ9uEHq8WBkiwus5clGS17Qd\r\n" \ + "dS/TOisB59tQruLx1E1bPLtBKyqk4koC5WAULJwfpswGSyWJTpYwIpxcWE3D2tBu\r\n" \ + "UB6MZfXZFzWmWEOyKbeoXjXe8GBCGgHLywvYDsGQ36HSGtEsAvR2QaTLSxWYcfk1\r\n" \ + "fbDn4jSWkb4yZy1r01UEigFQtONieGwRFaUqEcFJHJvEEGVgh9keaVlOj2vrwf5r\r\n" \ + "4mN4lW7gLdenN6g=\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is taken from tests/data_files/server2.crt.der. */ +/* BEGIN FILE binary macro TEST_SRV_CRT_RSA_SHA1_DER tests/data_files/server2.crt.der */ +#define TEST_SRV_CRT_RSA_SHA1_DER { \ + 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ + 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ + 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ + 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ + 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ + 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ + 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ + 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ + 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ + 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ + 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ + 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ + 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ + 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ + 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ + 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ + 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ + 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ + 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ + 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ + 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ + 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ + 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ + 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ + 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ + 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ + 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ + 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ + 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, \ + 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x01, 0x73, 0x0b, 0x4a, 0xc5, \ + 0xcb, 0xa0, 0xde, 0xf1, 0x63, 0x1c, 0x76, 0x04, 0x2b, 0x13, 0x0d, 0xc0, \ + 0x84, 0x11, 0xc5, 0x8f, 0x3a, 0xa7, 0xc5, 0x9c, 0x35, 0x7a, 0x77, 0xb8, \ + 0x20, 0x14, 0x82, 0xee, 0x54, 0xf0, 0xf2, 0xb0, 0x52, 0xcb, 0x78, 0xce, \ + 0x59, 0x07, 0x4f, 0x51, 0x69, 0xfe, 0xd3, 0x2f, 0xe9, 0x09, 0xe7, 0x85, \ + 0x92, 0xd8, 0xba, 0xb1, 0xeb, 0xc5, 0x76, 0x5d, 0x61, 0x2d, 0xe9, 0x86, \ + 0xb5, 0xde, 0x2a, 0xf9, 0x3f, 0x53, 0x28, 0x42, 0x86, 0x83, 0x73, 0x43, \ + 0xe0, 0x04, 0x5f, 0x07, 0x90, 0x14, 0x65, 0x9f, 0x6e, 0x10, 0x7a, 0xbc, \ + 0x58, 0x19, 0x22, 0xc2, 0xeb, 0x39, 0x72, 0x51, 0x92, 0xd7, 0xb4, 0x1d, \ + 0x75, 0x2f, 0xd3, 0x3a, 0x2b, 0x01, 0xe7, 0xdb, 0x50, 0xae, 0xe2, 0xf1, \ + 0xd4, 0x4d, 0x5b, 0x3c, 0xbb, 0x41, 0x2b, 0x2a, 0xa4, 0xe2, 0x4a, 0x02, \ + 0xe5, 0x60, 0x14, 0x2c, 0x9c, 0x1f, 0xa6, 0xcc, 0x06, 0x4b, 0x25, 0x89, \ + 0x4e, 0x96, 0x30, 0x22, 0x9c, 0x5c, 0x58, 0x4d, 0xc3, 0xda, 0xd0, 0x6e, \ + 0x50, 0x1e, 0x8c, 0x65, 0xf5, 0xd9, 0x17, 0x35, 0xa6, 0x58, 0x43, 0xb2, \ + 0x29, 0xb7, 0xa8, 0x5e, 0x35, 0xde, 0xf0, 0x60, 0x42, 0x1a, 0x01, 0xcb, \ + 0xcb, 0x0b, 0xd8, 0x0e, 0xc1, 0x90, 0xdf, 0xa1, 0xd2, 0x1a, 0xd1, 0x2c, \ + 0x02, 0xf4, 0x76, 0x41, 0xa4, 0xcb, 0x4b, 0x15, 0x98, 0x71, 0xf9, 0x35, \ + 0x7d, 0xb0, 0xe7, 0xe2, 0x34, 0x96, 0x91, 0xbe, 0x32, 0x67, 0x2d, 0x6b, \ + 0xd3, 0x55, 0x04, 0x8a, 0x01, 0x50, 0xb4, 0xe3, 0x62, 0x78, 0x6c, 0x11, \ + 0x15, 0xa5, 0x2a, 0x11, 0xc1, 0x49, 0x1c, 0x9b, 0xc4, 0x10, 0x65, 0x60, \ + 0x87, 0xd9, 0x1e, 0x69, 0x59, 0x4e, 0x8f, 0x6b, 0xeb, 0xc1, 0xfe, 0x6b, \ + 0xe2, 0x63, 0x78, 0x95, 0x6e, 0xe0, 0x2d, 0xd7, 0xa7, 0x37, 0xa8 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/server2.key. */ +/* BEGIN FILE string macro TEST_SRV_KEY_RSA_PEM tests/data_files/server2.key */ +#define TEST_SRV_KEY_RSA_PEM \ + "-----BEGIN RSA PRIVATE KEY-----\r\n" \ + "MIIEpAIBAAKCAQEAwU2j3efNHdEE10lyuJmsDnjkOjxKzzoTFtBa5M2jAIin7h5r\r\n" \ + "lqdStJDvLXJ6PiSa/LY0rCT1d+AmZIycsCh9odrqjObJHJa8/sEEUrM21KP64bF2\r\n" \ + "2JDBYbRmUjaiJlOqq3ReB30Zgtsq2B+g2Q0cLUlm91slc0boC4pPaQy1AJDh2oIQ\r\n" \ + "Zn2uVCuLZXmRoeJhw81ASQjuaAzxi4bSRr/QuKoRAx5/VqgaHkQYDw+Fi9qLRF7i\r\n" \ + "GMZiL8dmjfpd2H3zJ4kpAcWQDj8n8TDISg7v1t7HxydrxwU9esQCPJodPg/oNJhb\r\n" \ + "y3NLUpbYEaIsgIhpOVrTD7DeWS8Rx/fqEgEwlwIDAQABAoIBAQCXR0S8EIHFGORZ\r\n" \ + "++AtOg6eENxD+xVs0f1IeGz57Tjo3QnXX7VBZNdj+p1ECvhCE/G7XnkgU5hLZX+G\r\n" \ + "Z0jkz/tqJOI0vRSdLBbipHnWouyBQ4e/A1yIJdlBtqXxJ1KE/ituHRbNc4j4kL8Z\r\n" \ + "/r6pvwnTI0PSx2Eqs048YdS92LT6qAv4flbNDxMn2uY7s4ycS4Q8w1JXnCeaAnYm\r\n" \ + "WYI5wxO+bvRELR2Mcz5DmVnL8jRyml6l6582bSv5oufReFIbyPZbQWlXgYnpu6He\r\n" \ + "GTc7E1zKYQGG/9+DQUl/1vQuCPqQwny0tQoX2w5tdYpdMdVm+zkLtbajzdTviJJa\r\n" \ + "TWzL6lt5AoGBAN86+SVeJDcmQJcv4Eq6UhtRr4QGMiQMz0Sod6ettYxYzMgxtw28\r\n" \ + "CIrgpozCc+UaZJLo7UxvC6an85r1b2nKPCLQFaggJ0H4Q0J/sZOhBIXaoBzWxveK\r\n" \ + "nupceKdVxGsFi8CDy86DBfiyFivfBj+47BbaQzPBj7C4rK7UlLjab2rDAoGBAN2u\r\n" \ + "AM2gchoFiu4v1HFL8D7lweEpi6ZnMJjnEu/dEgGQJFjwdpLnPbsj4c75odQ4Gz8g\r\n" \ + "sw9lao9VVzbusoRE/JGI4aTdO0pATXyG7eG1Qu+5Yc1YGXcCrliA2xM9xx+d7f+s\r\n" \ + "mPzN+WIEg5GJDYZDjAzHG5BNvi/FfM1C9dOtjv2dAoGAF0t5KmwbjWHBhcVqO4Ic\r\n" \ + "BVvN3BIlc1ue2YRXEDlxY5b0r8N4XceMgKmW18OHApZxfl8uPDauWZLXOgl4uepv\r\n" \ + "whZC3EuWrSyyICNhLY21Ah7hbIEBPF3L3ZsOwC+UErL+dXWLdB56Jgy3gZaBeW7b\r\n" \ + "vDrEnocJbqCm7IukhXHOBK8CgYEAwqdHB0hqyNSzIOGY7v9abzB6pUdA3BZiQvEs\r\n" \ + "3LjHVd4HPJ2x0N8CgrBIWOE0q8+0hSMmeE96WW/7jD3fPWwCR5zlXknxBQsfv0gP\r\n" \ + "3BC5PR0Qdypz+d+9zfMf625kyit4T/hzwhDveZUzHnk1Cf+IG7Q+TOEnLnWAWBED\r\n" \ + "ISOWmrUCgYAFEmRxgwAc/u+D6t0syCwAYh6POtscq9Y0i9GyWk89NzgC4NdwwbBH\r\n" \ + "4AgahOxIxXx2gxJnq3yfkJfIjwf0s2DyP0kY2y6Ua1OeomPeY9mrIS4tCuDQ6LrE\r\n" \ + "TB6l9VGoxJL4fyHnZb8L5gGvnB1bbD8cL6YPaDiOhcRseC9vBiEuVg==\r\n" \ + "-----END RSA PRIVATE KEY-----\r\n" +/* END FILE */ + +/* This was generated from tests/data_files/server2.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_SRV_KEY_RSA_DER tests/data_files/server2.key.der */ +#define TEST_SRV_KEY_RSA_DER { \ + 0x30, 0x82, 0x04, 0xa4, 0x02, 0x01, 0x00, 0x02, 0x82, 0x01, 0x01, 0x00, \ + 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, \ + 0xb8, 0x99, 0xac, 0x0e, 0x78, 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, \ + 0x16, 0xd0, 0x5a, 0xe4, 0xcd, 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, \ + 0x96, 0xa7, 0x52, 0xb4, 0x90, 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, \ + 0xfc, 0xb6, 0x34, 0xac, 0x24, 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, \ + 0xb0, 0x28, 0x7d, 0xa1, 0xda, 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, \ + 0xfe, 0xc1, 0x04, 0x52, 0xb3, 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, \ + 0xd8, 0x90, 0xc1, 0x61, 0xb4, 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, \ + 0xab, 0x74, 0x5e, 0x07, 0x7d, 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, \ + 0xd9, 0x0d, 0x1c, 0x2d, 0x49, 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, \ + 0x0b, 0x8a, 0x4f, 0x69, 0x0c, 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, \ + 0x66, 0x7d, 0xae, 0x54, 0x2b, 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, \ + 0xc3, 0xcd, 0x40, 0x49, 0x08, 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, \ + 0x46, 0xbf, 0xd0, 0xb8, 0xaa, 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, \ + 0x1e, 0x44, 0x18, 0x0f, 0x0f, 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, \ + 0x18, 0xc6, 0x62, 0x2f, 0xc7, 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, \ + 0x27, 0x89, 0x29, 0x01, 0xc5, 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, \ + 0x4a, 0x0e, 0xef, 0xd6, 0xde, 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, \ + 0x7a, 0xc4, 0x02, 0x3c, 0x9a, 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, \ + 0xcb, 0x73, 0x4b, 0x52, 0x96, 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, \ + 0x39, 0x5a, 0xd3, 0x0f, 0xb0, 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, \ + 0x12, 0x01, 0x30, 0x97, 0x02, 0x03, 0x01, 0x00, 0x01, 0x02, 0x82, 0x01, \ + 0x01, 0x00, 0x97, 0x47, 0x44, 0xbc, 0x10, 0x81, 0xc5, 0x18, 0xe4, 0x59, \ + 0xfb, 0xe0, 0x2d, 0x3a, 0x0e, 0x9e, 0x10, 0xdc, 0x43, 0xfb, 0x15, 0x6c, \ + 0xd1, 0xfd, 0x48, 0x78, 0x6c, 0xf9, 0xed, 0x38, 0xe8, 0xdd, 0x09, 0xd7, \ + 0x5f, 0xb5, 0x41, 0x64, 0xd7, 0x63, 0xfa, 0x9d, 0x44, 0x0a, 0xf8, 0x42, \ + 0x13, 0xf1, 0xbb, 0x5e, 0x79, 0x20, 0x53, 0x98, 0x4b, 0x65, 0x7f, 0x86, \ + 0x67, 0x48, 0xe4, 0xcf, 0xfb, 0x6a, 0x24, 0xe2, 0x34, 0xbd, 0x14, 0x9d, \ + 0x2c, 0x16, 0xe2, 0xa4, 0x79, 0xd6, 0xa2, 0xec, 0x81, 0x43, 0x87, 0xbf, \ + 0x03, 0x5c, 0x88, 0x25, 0xd9, 0x41, 0xb6, 0xa5, 0xf1, 0x27, 0x52, 0x84, \ + 0xfe, 0x2b, 0x6e, 0x1d, 0x16, 0xcd, 0x73, 0x88, 0xf8, 0x90, 0xbf, 0x19, \ + 0xfe, 0xbe, 0xa9, 0xbf, 0x09, 0xd3, 0x23, 0x43, 0xd2, 0xc7, 0x61, 0x2a, \ + 0xb3, 0x4e, 0x3c, 0x61, 0xd4, 0xbd, 0xd8, 0xb4, 0xfa, 0xa8, 0x0b, 0xf8, \ + 0x7e, 0x56, 0xcd, 0x0f, 0x13, 0x27, 0xda, 0xe6, 0x3b, 0xb3, 0x8c, 0x9c, \ + 0x4b, 0x84, 0x3c, 0xc3, 0x52, 0x57, 0x9c, 0x27, 0x9a, 0x02, 0x76, 0x26, \ + 0x59, 0x82, 0x39, 0xc3, 0x13, 0xbe, 0x6e, 0xf4, 0x44, 0x2d, 0x1d, 0x8c, \ + 0x73, 0x3e, 0x43, 0x99, 0x59, 0xcb, 0xf2, 0x34, 0x72, 0x9a, 0x5e, 0xa5, \ + 0xeb, 0x9f, 0x36, 0x6d, 0x2b, 0xf9, 0xa2, 0xe7, 0xd1, 0x78, 0x52, 0x1b, \ + 0xc8, 0xf6, 0x5b, 0x41, 0x69, 0x57, 0x81, 0x89, 0xe9, 0xbb, 0xa1, 0xde, \ + 0x19, 0x37, 0x3b, 0x13, 0x5c, 0xca, 0x61, 0x01, 0x86, 0xff, 0xdf, 0x83, \ + 0x41, 0x49, 0x7f, 0xd6, 0xf4, 0x2e, 0x08, 0xfa, 0x90, 0xc2, 0x7c, 0xb4, \ + 0xb5, 0x0a, 0x17, 0xdb, 0x0e, 0x6d, 0x75, 0x8a, 0x5d, 0x31, 0xd5, 0x66, \ + 0xfb, 0x39, 0x0b, 0xb5, 0xb6, 0xa3, 0xcd, 0xd4, 0xef, 0x88, 0x92, 0x5a, \ + 0x4d, 0x6c, 0xcb, 0xea, 0x5b, 0x79, 0x02, 0x81, 0x81, 0x00, 0xdf, 0x3a, \ + 0xf9, 0x25, 0x5e, 0x24, 0x37, 0x26, 0x40, 0x97, 0x2f, 0xe0, 0x4a, 0xba, \ + 0x52, 0x1b, 0x51, 0xaf, 0x84, 0x06, 0x32, 0x24, 0x0c, 0xcf, 0x44, 0xa8, \ + 0x77, 0xa7, 0xad, 0xb5, 0x8c, 0x58, 0xcc, 0xc8, 0x31, 0xb7, 0x0d, 0xbc, \ + 0x08, 0x8a, 0xe0, 0xa6, 0x8c, 0xc2, 0x73, 0xe5, 0x1a, 0x64, 0x92, 0xe8, \ + 0xed, 0x4c, 0x6f, 0x0b, 0xa6, 0xa7, 0xf3, 0x9a, 0xf5, 0x6f, 0x69, 0xca, \ + 0x3c, 0x22, 0xd0, 0x15, 0xa8, 0x20, 0x27, 0x41, 0xf8, 0x43, 0x42, 0x7f, \ + 0xb1, 0x93, 0xa1, 0x04, 0x85, 0xda, 0xa0, 0x1c, 0xd6, 0xc6, 0xf7, 0x8a, \ + 0x9e, 0xea, 0x5c, 0x78, 0xa7, 0x55, 0xc4, 0x6b, 0x05, 0x8b, 0xc0, 0x83, \ + 0xcb, 0xce, 0x83, 0x05, 0xf8, 0xb2, 0x16, 0x2b, 0xdf, 0x06, 0x3f, 0xb8, \ + 0xec, 0x16, 0xda, 0x43, 0x33, 0xc1, 0x8f, 0xb0, 0xb8, 0xac, 0xae, 0xd4, \ + 0x94, 0xb8, 0xda, 0x6f, 0x6a, 0xc3, 0x02, 0x81, 0x81, 0x00, 0xdd, 0xae, \ + 0x00, 0xcd, 0xa0, 0x72, 0x1a, 0x05, 0x8a, 0xee, 0x2f, 0xd4, 0x71, 0x4b, \ + 0xf0, 0x3e, 0xe5, 0xc1, 0xe1, 0x29, 0x8b, 0xa6, 0x67, 0x30, 0x98, 0xe7, \ + 0x12, 0xef, 0xdd, 0x12, 0x01, 0x90, 0x24, 0x58, 0xf0, 0x76, 0x92, 0xe7, \ + 0x3d, 0xbb, 0x23, 0xe1, 0xce, 0xf9, 0xa1, 0xd4, 0x38, 0x1b, 0x3f, 0x20, \ + 0xb3, 0x0f, 0x65, 0x6a, 0x8f, 0x55, 0x57, 0x36, 0xee, 0xb2, 0x84, 0x44, \ + 0xfc, 0x91, 0x88, 0xe1, 0xa4, 0xdd, 0x3b, 0x4a, 0x40, 0x4d, 0x7c, 0x86, \ + 0xed, 0xe1, 0xb5, 0x42, 0xef, 0xb9, 0x61, 0xcd, 0x58, 0x19, 0x77, 0x02, \ + 0xae, 0x58, 0x80, 0xdb, 0x13, 0x3d, 0xc7, 0x1f, 0x9d, 0xed, 0xff, 0xac, \ + 0x98, 0xfc, 0xcd, 0xf9, 0x62, 0x04, 0x83, 0x91, 0x89, 0x0d, 0x86, 0x43, \ + 0x8c, 0x0c, 0xc7, 0x1b, 0x90, 0x4d, 0xbe, 0x2f, 0xc5, 0x7c, 0xcd, 0x42, \ + 0xf5, 0xd3, 0xad, 0x8e, 0xfd, 0x9d, 0x02, 0x81, 0x80, 0x17, 0x4b, 0x79, \ + 0x2a, 0x6c, 0x1b, 0x8d, 0x61, 0xc1, 0x85, 0xc5, 0x6a, 0x3b, 0x82, 0x1c, \ + 0x05, 0x5b, 0xcd, 0xdc, 0x12, 0x25, 0x73, 0x5b, 0x9e, 0xd9, 0x84, 0x57, \ + 0x10, 0x39, 0x71, 0x63, 0x96, 0xf4, 0xaf, 0xc3, 0x78, 0x5d, 0xc7, 0x8c, \ + 0x80, 0xa9, 0x96, 0xd7, 0xc3, 0x87, 0x02, 0x96, 0x71, 0x7e, 0x5f, 0x2e, \ + 0x3c, 0x36, 0xae, 0x59, 0x92, 0xd7, 0x3a, 0x09, 0x78, 0xb9, 0xea, 0x6f, \ + 0xc2, 0x16, 0x42, 0xdc, 0x4b, 0x96, 0xad, 0x2c, 0xb2, 0x20, 0x23, 0x61, \ + 0x2d, 0x8d, 0xb5, 0x02, 0x1e, 0xe1, 0x6c, 0x81, 0x01, 0x3c, 0x5d, 0xcb, \ + 0xdd, 0x9b, 0x0e, 0xc0, 0x2f, 0x94, 0x12, 0xb2, 0xfe, 0x75, 0x75, 0x8b, \ + 0x74, 0x1e, 0x7a, 0x26, 0x0c, 0xb7, 0x81, 0x96, 0x81, 0x79, 0x6e, 0xdb, \ + 0xbc, 0x3a, 0xc4, 0x9e, 0x87, 0x09, 0x6e, 0xa0, 0xa6, 0xec, 0x8b, 0xa4, \ + 0x85, 0x71, 0xce, 0x04, 0xaf, 0x02, 0x81, 0x81, 0x00, 0xc2, 0xa7, 0x47, \ + 0x07, 0x48, 0x6a, 0xc8, 0xd4, 0xb3, 0x20, 0xe1, 0x98, 0xee, 0xff, 0x5a, \ + 0x6f, 0x30, 0x7a, 0xa5, 0x47, 0x40, 0xdc, 0x16, 0x62, 0x42, 0xf1, 0x2c, \ + 0xdc, 0xb8, 0xc7, 0x55, 0xde, 0x07, 0x3c, 0x9d, 0xb1, 0xd0, 0xdf, 0x02, \ + 0x82, 0xb0, 0x48, 0x58, 0xe1, 0x34, 0xab, 0xcf, 0xb4, 0x85, 0x23, 0x26, \ + 0x78, 0x4f, 0x7a, 0x59, 0x6f, 0xfb, 0x8c, 0x3d, 0xdf, 0x3d, 0x6c, 0x02, \ + 0x47, 0x9c, 0xe5, 0x5e, 0x49, 0xf1, 0x05, 0x0b, 0x1f, 0xbf, 0x48, 0x0f, \ + 0xdc, 0x10, 0xb9, 0x3d, 0x1d, 0x10, 0x77, 0x2a, 0x73, 0xf9, 0xdf, 0xbd, \ + 0xcd, 0xf3, 0x1f, 0xeb, 0x6e, 0x64, 0xca, 0x2b, 0x78, 0x4f, 0xf8, 0x73, \ + 0xc2, 0x10, 0xef, 0x79, 0x95, 0x33, 0x1e, 0x79, 0x35, 0x09, 0xff, 0x88, \ + 0x1b, 0xb4, 0x3e, 0x4c, 0xe1, 0x27, 0x2e, 0x75, 0x80, 0x58, 0x11, 0x03, \ + 0x21, 0x23, 0x96, 0x9a, 0xb5, 0x02, 0x81, 0x80, 0x05, 0x12, 0x64, 0x71, \ + 0x83, 0x00, 0x1c, 0xfe, 0xef, 0x83, 0xea, 0xdd, 0x2c, 0xc8, 0x2c, 0x00, \ + 0x62, 0x1e, 0x8f, 0x3a, 0xdb, 0x1c, 0xab, 0xd6, 0x34, 0x8b, 0xd1, 0xb2, \ + 0x5a, 0x4f, 0x3d, 0x37, 0x38, 0x02, 0xe0, 0xd7, 0x70, 0xc1, 0xb0, 0x47, \ + 0xe0, 0x08, 0x1a, 0x84, 0xec, 0x48, 0xc5, 0x7c, 0x76, 0x83, 0x12, 0x67, \ + 0xab, 0x7c, 0x9f, 0x90, 0x97, 0xc8, 0x8f, 0x07, 0xf4, 0xb3, 0x60, 0xf2, \ + 0x3f, 0x49, 0x18, 0xdb, 0x2e, 0x94, 0x6b, 0x53, 0x9e, 0xa2, 0x63, 0xde, \ + 0x63, 0xd9, 0xab, 0x21, 0x2e, 0x2d, 0x0a, 0xe0, 0xd0, 0xe8, 0xba, 0xc4, \ + 0x4c, 0x1e, 0xa5, 0xf5, 0x51, 0xa8, 0xc4, 0x92, 0xf8, 0x7f, 0x21, 0xe7, \ + 0x65, 0xbf, 0x0b, 0xe6, 0x01, 0xaf, 0x9c, 0x1d, 0x5b, 0x6c, 0x3f, 0x1c, \ + 0x2f, 0xa6, 0x0f, 0x68, 0x38, 0x8e, 0x85, 0xc4, 0x6c, 0x78, 0x2f, 0x6f, \ + 0x06, 0x21, 0x2e, 0x56 \ +} +/* END FILE */ + +/* + * Test client Certificates + * + * Test client certificates are defined for each choice + * of the following parameters: + * - PEM or DER encoding + * - RSA or EC key + * + * Things to add: + * - hash type + * - multiple EC curve types + */ + +/* This is taken from tests/data_files/cli2.crt. */ +/* BEGIN FILE string macro TEST_CLI_CRT_EC_PEM tests/data_files/cli2.crt */ +#define TEST_CLI_CRT_EC_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIICLDCCAbKgAwIBAgIBDTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\r\n" \ + "MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjBBMQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UEChMIUG9sYXJTU0wxHzAdBgNVBAMTFlBvbGFyU1NMIFRlc3QgQ2xpZW50IDIw\r\n" \ + "WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARX5a6xc9/TrLuTuIH/Eq7u5lOszlVT\r\n" \ + "9jQOzC7jYyUL35ji81xgNpbA1RgUcOV/n9VLRRjlsGzVXPiWj4dwo+THo4GdMIGa\r\n" \ + "MAkGA1UdEwQCMAAwHQYDVR0OBBYEFHoAX4Zk/OBd5REQO7LmO8QmP8/iMG4GA1Ud\r\n" \ + "IwRnMGWAFJ1tICRJAT8ry3i1Gbx+JMnb+zZ8oUKkQDA+MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0GC\r\n" \ + "CQDBQ+J+YkPM6DAKBggqhkjOPQQDAgNoADBlAjBKZQ17IIOimbmoD/yN7o89u3BM\r\n" \ + "lgOsjnhw3fIOoLIWy2WOGsk/LGF++DzvrRzuNiACMQCd8iem1XS4JK7haj8xocpU\r\n" \ + "LwjQje5PDGHfd3h9tP38Qknu5bJqws0md2KOKHyeV0U=\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This is generated from tests/data_files/cli2.crt.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CLI_CRT_EC_DER tests/data_files/cli2.crt.der */ +#define TEST_CLI_CRT_EC_DER { \ + 0x30, 0x82, 0x02, 0x2c, 0x30, 0x82, 0x01, 0xb2, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x0d, 0x30, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, \ + 0x3d, 0x04, 0x03, 0x02, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ + 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x33, 0x30, 0x39, 0x32, 0x34, 0x31, 0x35, 0x35, 0x32, 0x30, 0x34, \ + 0x5a, 0x17, 0x0d, 0x32, 0x33, 0x30, 0x39, 0x32, 0x32, 0x31, 0x35, 0x35, \ + 0x32, 0x30, 0x34, 0x5a, 0x30, 0x41, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x1f, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ + 0x16, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x32, 0x30, \ + 0x59, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, \ + 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07, 0x03, 0x42, \ + 0x00, 0x04, 0x57, 0xe5, 0xae, 0xb1, 0x73, 0xdf, 0xd3, 0xac, 0xbb, 0x93, \ + 0xb8, 0x81, 0xff, 0x12, 0xae, 0xee, 0xe6, 0x53, 0xac, 0xce, 0x55, 0x53, \ + 0xf6, 0x34, 0x0e, 0xcc, 0x2e, 0xe3, 0x63, 0x25, 0x0b, 0xdf, 0x98, 0xe2, \ + 0xf3, 0x5c, 0x60, 0x36, 0x96, 0xc0, 0xd5, 0x18, 0x14, 0x70, 0xe5, 0x7f, \ + 0x9f, 0xd5, 0x4b, 0x45, 0x18, 0xe5, 0xb0, 0x6c, 0xd5, 0x5c, 0xf8, 0x96, \ + 0x8f, 0x87, 0x70, 0xa3, 0xe4, 0xc7, 0xa3, 0x81, 0x9d, 0x30, 0x81, 0x9a, \ + 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, \ + 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x7a, 0x00, \ + 0x5f, 0x86, 0x64, 0xfc, 0xe0, 0x5d, 0xe5, 0x11, 0x10, 0x3b, 0xb2, 0xe6, \ + 0x3b, 0xc4, 0x26, 0x3f, 0xcf, 0xe2, 0x30, 0x6e, 0x06, 0x03, 0x55, 0x1d, \ + 0x23, 0x04, 0x67, 0x30, 0x65, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, \ + 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, \ + 0xfb, 0x36, 0x7c, 0xa1, 0x42, 0xa4, 0x40, 0x30, 0x3e, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x82, \ + 0x09, 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, 0x30, 0x0a, \ + 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, 0x03, 0x68, \ + 0x00, 0x30, 0x65, 0x02, 0x30, 0x4a, 0x65, 0x0d, 0x7b, 0x20, 0x83, 0xa2, \ + 0x99, 0xb9, 0xa8, 0x0f, 0xfc, 0x8d, 0xee, 0x8f, 0x3d, 0xbb, 0x70, 0x4c, \ + 0x96, 0x03, 0xac, 0x8e, 0x78, 0x70, 0xdd, 0xf2, 0x0e, 0xa0, 0xb2, 0x16, \ + 0xcb, 0x65, 0x8e, 0x1a, 0xc9, 0x3f, 0x2c, 0x61, 0x7e, 0xf8, 0x3c, 0xef, \ + 0xad, 0x1c, 0xee, 0x36, 0x20, 0x02, 0x31, 0x00, 0x9d, 0xf2, 0x27, 0xa6, \ + 0xd5, 0x74, 0xb8, 0x24, 0xae, 0xe1, 0x6a, 0x3f, 0x31, 0xa1, 0xca, 0x54, \ + 0x2f, 0x08, 0xd0, 0x8d, 0xee, 0x4f, 0x0c, 0x61, 0xdf, 0x77, 0x78, 0x7d, \ + 0xb4, 0xfd, 0xfc, 0x42, 0x49, 0xee, 0xe5, 0xb2, 0x6a, 0xc2, 0xcd, 0x26, \ + 0x77, 0x62, 0x8e, 0x28, 0x7c, 0x9e, 0x57, 0x45 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/cli2.key. */ +/* BEGIN FILE string macro TEST_CLI_KEY_EC_PEM tests/data_files/cli2.key */ +#define TEST_CLI_KEY_EC_PEM \ + "-----BEGIN EC PRIVATE KEY-----\r\n" \ + "MHcCAQEEIPb3hmTxZ3/mZI3vyk7p3U3wBf+WIop6hDhkFzJhmLcqoAoGCCqGSM49\r\n" \ + "AwEHoUQDQgAEV+WusXPf06y7k7iB/xKu7uZTrM5VU/Y0Dswu42MlC9+Y4vNcYDaW\r\n" \ + "wNUYFHDlf5/VS0UY5bBs1Vz4lo+HcKPkxw==\r\n" \ + "-----END EC PRIVATE KEY-----\r\n" +/* END FILE */ + +/* This is generated from tests/data_files/cli2.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CLI_KEY_EC_DER tests/data_files/cli2.key.der */ +#define TEST_CLI_KEY_EC_DER { \ + 0x30, 0x77, 0x02, 0x01, 0x01, 0x04, 0x20, 0xf6, 0xf7, 0x86, 0x64, 0xf1, \ + 0x67, 0x7f, 0xe6, 0x64, 0x8d, 0xef, 0xca, 0x4e, 0xe9, 0xdd, 0x4d, 0xf0, \ + 0x05, 0xff, 0x96, 0x22, 0x8a, 0x7a, 0x84, 0x38, 0x64, 0x17, 0x32, 0x61, \ + 0x98, 0xb7, 0x2a, 0xa0, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, \ + 0x03, 0x01, 0x07, 0xa1, 0x44, 0x03, 0x42, 0x00, 0x04, 0x57, 0xe5, 0xae, \ + 0xb1, 0x73, 0xdf, 0xd3, 0xac, 0xbb, 0x93, 0xb8, 0x81, 0xff, 0x12, 0xae, \ + 0xee, 0xe6, 0x53, 0xac, 0xce, 0x55, 0x53, 0xf6, 0x34, 0x0e, 0xcc, 0x2e, \ + 0xe3, 0x63, 0x25, 0x0b, 0xdf, 0x98, 0xe2, 0xf3, 0x5c, 0x60, 0x36, 0x96, \ + 0xc0, 0xd5, 0x18, 0x14, 0x70, 0xe5, 0x7f, 0x9f, 0xd5, 0x4b, 0x45, 0x18, \ + 0xe5, 0xb0, 0x6c, 0xd5, 0x5c, 0xf8, 0x96, 0x8f, 0x87, 0x70, 0xa3, 0xe4, \ + 0xc7 \ +} +/* END FILE */ + +/* This is taken from tests/data_files/cli-rsa-sha256.crt. */ +/* BEGIN FILE string macro TEST_CLI_CRT_RSA_PEM tests/data_files/cli-rsa-sha256.crt */ +#define TEST_CLI_CRT_RSA_PEM \ + "-----BEGIN CERTIFICATE-----\r\n" \ + "MIIDPzCCAiegAwIBAgIBBDANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ + "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ + "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA8MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "A1UECgwIUG9sYXJTU0wxGjAYBgNVBAMMEVBvbGFyU1NMIENsaWVudCAyMIIBIjAN\r\n" \ + "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyHTEzLn5tXnpRdkUYLB9u5Pyax6f\r\n" \ + "M60Nj4o8VmXl3ETZzGaFB9X4J7BKNdBjngpuG7fa8H6r7gwQk4ZJGDTzqCrSV/Uu\r\n" \ + "1C93KYRhTYJQj6eVSHD1bk2y1RPD0hrt5kPqQhTrdOrA7R/UV06p86jt0uDBMHEw\r\n" \ + "MjDV0/YI0FZPRo7yX/k9Z5GIMC5Cst99++UMd//sMcB4j7/Cf8qtbCHWjdmLao5v\r\n" \ + "4Jv4EFbMs44TFeY0BGbH7vk2DmqV9gmaBmf0ZXH4yqSxJeD+PIs1BGe64E92hfx/\r\n" \ + "/DZrtenNLQNiTrM9AM+vdqBpVoNq0qjU51Bx5rU2BXcFbXvI5MT9TNUhXwIDAQAB\r\n" \ + "o00wSzAJBgNVHRMEAjAAMB0GA1UdDgQWBBRxoQBzckAvVHZeM/xSj7zx3WtGITAf\r\n" \ + "BgNVHSMEGDAWgBS0WuSls97SUva51aaVD+s+vMf9/zANBgkqhkiG9w0BAQsFAAOC\r\n" \ + "AQEAlHabem2Tu69VUN7EipwnQn1dIHdgvT5i+iQHpSxY1crPnBbAeSdAXwsVEqLQ\r\n" \ + "gOOIAQD5VIITNuoGgo4i+4OpNh9u7ZkpRHla+/swsfrFWRRbBNP5Bcu74AGLstwU\r\n" \ + "zM8gIkBiyfM1Q1qDQISV9trlCG6O8vh8dp/rbI3rfzo99BOHXgFCrzXjCuW4vDsF\r\n" \ + "r+Dao26bX3sJ6UnEWg1H3o2x6PpUcvQ36h71/bz4TEbbUUEpe02V4QWuL+wrhHJL\r\n" \ + "U7o3SVE3Og7jPF8sat0a50YUWhwEFI256m02KAXLg89ueUyYKEr6rNwhcvXJpvU9\r\n" \ + "giIVvd0Sbjjnn7NC4VDbcXV8vw==\r\n" \ + "-----END CERTIFICATE-----\r\n" +/* END FILE */ + +/* This was generated from tests/data_files/cli-rsa-sha256.crt.der + using `xxd -i.` */ +/* BEGIN FILE binary macro TEST_CLI_CRT_RSA_DER tests/data_files/cli-rsa-sha256.crt.der */ +#define TEST_CLI_CRT_RSA_DER { \ + 0x30, 0x82, 0x03, 0x3f, 0x30, 0x82, 0x02, 0x27, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x04, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x3c, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x11, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x43, 0x6c, \ + 0x69, 0x65, 0x6e, 0x74, 0x20, 0x32, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, \ + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, \ + 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, \ + 0x01, 0x01, 0x00, 0xc8, 0x74, 0xc4, 0xcc, 0xb9, 0xf9, 0xb5, 0x79, 0xe9, \ + 0x45, 0xd9, 0x14, 0x60, 0xb0, 0x7d, 0xbb, 0x93, 0xf2, 0x6b, 0x1e, 0x9f, \ + 0x33, 0xad, 0x0d, 0x8f, 0x8a, 0x3c, 0x56, 0x65, 0xe5, 0xdc, 0x44, 0xd9, \ + 0xcc, 0x66, 0x85, 0x07, 0xd5, 0xf8, 0x27, 0xb0, 0x4a, 0x35, 0xd0, 0x63, \ + 0x9e, 0x0a, 0x6e, 0x1b, 0xb7, 0xda, 0xf0, 0x7e, 0xab, 0xee, 0x0c, 0x10, \ + 0x93, 0x86, 0x49, 0x18, 0x34, 0xf3, 0xa8, 0x2a, 0xd2, 0x57, 0xf5, 0x2e, \ + 0xd4, 0x2f, 0x77, 0x29, 0x84, 0x61, 0x4d, 0x82, 0x50, 0x8f, 0xa7, 0x95, \ + 0x48, 0x70, 0xf5, 0x6e, 0x4d, 0xb2, 0xd5, 0x13, 0xc3, 0xd2, 0x1a, 0xed, \ + 0xe6, 0x43, 0xea, 0x42, 0x14, 0xeb, 0x74, 0xea, 0xc0, 0xed, 0x1f, 0xd4, \ + 0x57, 0x4e, 0xa9, 0xf3, 0xa8, 0xed, 0xd2, 0xe0, 0xc1, 0x30, 0x71, 0x30, \ + 0x32, 0x30, 0xd5, 0xd3, 0xf6, 0x08, 0xd0, 0x56, 0x4f, 0x46, 0x8e, 0xf2, \ + 0x5f, 0xf9, 0x3d, 0x67, 0x91, 0x88, 0x30, 0x2e, 0x42, 0xb2, 0xdf, 0x7d, \ + 0xfb, 0xe5, 0x0c, 0x77, 0xff, 0xec, 0x31, 0xc0, 0x78, 0x8f, 0xbf, 0xc2, \ + 0x7f, 0xca, 0xad, 0x6c, 0x21, 0xd6, 0x8d, 0xd9, 0x8b, 0x6a, 0x8e, 0x6f, \ + 0xe0, 0x9b, 0xf8, 0x10, 0x56, 0xcc, 0xb3, 0x8e, 0x13, 0x15, 0xe6, 0x34, \ + 0x04, 0x66, 0xc7, 0xee, 0xf9, 0x36, 0x0e, 0x6a, 0x95, 0xf6, 0x09, 0x9a, \ + 0x06, 0x67, 0xf4, 0x65, 0x71, 0xf8, 0xca, 0xa4, 0xb1, 0x25, 0xe0, 0xfe, \ + 0x3c, 0x8b, 0x35, 0x04, 0x67, 0xba, 0xe0, 0x4f, 0x76, 0x85, 0xfc, 0x7f, \ + 0xfc, 0x36, 0x6b, 0xb5, 0xe9, 0xcd, 0x2d, 0x03, 0x62, 0x4e, 0xb3, 0x3d, \ + 0x00, 0xcf, 0xaf, 0x76, 0xa0, 0x69, 0x56, 0x83, 0x6a, 0xd2, 0xa8, 0xd4, \ + 0xe7, 0x50, 0x71, 0xe6, 0xb5, 0x36, 0x05, 0x77, 0x05, 0x6d, 0x7b, 0xc8, \ + 0xe4, 0xc4, 0xfd, 0x4c, 0xd5, 0x21, 0x5f, 0x02, 0x03, 0x01, 0x00, 0x01, \ + 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, \ + 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, \ + 0x04, 0x14, 0x71, 0xa1, 0x00, 0x73, 0x72, 0x40, 0x2f, 0x54, 0x76, 0x5e, \ + 0x33, 0xfc, 0x52, 0x8f, 0xbc, 0xf1, 0xdd, 0x6b, 0x46, 0x21, 0x30, 0x1f, \ + 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0xb4, \ + 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, 0xa6, 0x95, \ + 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, 0x09, 0x2a, \ + 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x03, 0x82, \ + 0x01, 0x01, 0x00, 0x94, 0x76, 0x9b, 0x7a, 0x6d, 0x93, 0xbb, 0xaf, 0x55, \ + 0x50, 0xde, 0xc4, 0x8a, 0x9c, 0x27, 0x42, 0x7d, 0x5d, 0x20, 0x77, 0x60, \ + 0xbd, 0x3e, 0x62, 0xfa, 0x24, 0x07, 0xa5, 0x2c, 0x58, 0xd5, 0xca, 0xcf, \ + 0x9c, 0x16, 0xc0, 0x79, 0x27, 0x40, 0x5f, 0x0b, 0x15, 0x12, 0xa2, 0xd0, \ + 0x80, 0xe3, 0x88, 0x01, 0x00, 0xf9, 0x54, 0x82, 0x13, 0x36, 0xea, 0x06, \ + 0x82, 0x8e, 0x22, 0xfb, 0x83, 0xa9, 0x36, 0x1f, 0x6e, 0xed, 0x99, 0x29, \ + 0x44, 0x79, 0x5a, 0xfb, 0xfb, 0x30, 0xb1, 0xfa, 0xc5, 0x59, 0x14, 0x5b, \ + 0x04, 0xd3, 0xf9, 0x05, 0xcb, 0xbb, 0xe0, 0x01, 0x8b, 0xb2, 0xdc, 0x14, \ + 0xcc, 0xcf, 0x20, 0x22, 0x40, 0x62, 0xc9, 0xf3, 0x35, 0x43, 0x5a, 0x83, \ + 0x40, 0x84, 0x95, 0xf6, 0xda, 0xe5, 0x08, 0x6e, 0x8e, 0xf2, 0xf8, 0x7c, \ + 0x76, 0x9f, 0xeb, 0x6c, 0x8d, 0xeb, 0x7f, 0x3a, 0x3d, 0xf4, 0x13, 0x87, \ + 0x5e, 0x01, 0x42, 0xaf, 0x35, 0xe3, 0x0a, 0xe5, 0xb8, 0xbc, 0x3b, 0x05, \ + 0xaf, 0xe0, 0xda, 0xa3, 0x6e, 0x9b, 0x5f, 0x7b, 0x09, 0xe9, 0x49, 0xc4, \ + 0x5a, 0x0d, 0x47, 0xde, 0x8d, 0xb1, 0xe8, 0xfa, 0x54, 0x72, 0xf4, 0x37, \ + 0xea, 0x1e, 0xf5, 0xfd, 0xbc, 0xf8, 0x4c, 0x46, 0xdb, 0x51, 0x41, 0x29, \ + 0x7b, 0x4d, 0x95, 0xe1, 0x05, 0xae, 0x2f, 0xec, 0x2b, 0x84, 0x72, 0x4b, \ + 0x53, 0xba, 0x37, 0x49, 0x51, 0x37, 0x3a, 0x0e, 0xe3, 0x3c, 0x5f, 0x2c, \ + 0x6a, 0xdd, 0x1a, 0xe7, 0x46, 0x14, 0x5a, 0x1c, 0x04, 0x14, 0x8d, 0xb9, \ + 0xea, 0x6d, 0x36, 0x28, 0x05, 0xcb, 0x83, 0xcf, 0x6e, 0x79, 0x4c, 0x98, \ + 0x28, 0x4a, 0xfa, 0xac, 0xdc, 0x21, 0x72, 0xf5, 0xc9, 0xa6, 0xf5, 0x3d, \ + 0x82, 0x22, 0x15, 0xbd, 0xdd, 0x12, 0x6e, 0x38, 0xe7, 0x9f, 0xb3, 0x42, \ + 0xe1, 0x50, 0xdb, 0x71, 0x75, 0x7c, 0xbf \ +} +/* END FILE */ + +/* This is taken from tests/data_files/cli-rsa.key. */ +/* BEGIN FILE string macro TEST_CLI_KEY_RSA_PEM tests/data_files/cli-rsa.key */ +#define TEST_CLI_KEY_RSA_PEM \ + "-----BEGIN RSA PRIVATE KEY-----\r\n" \ + "MIIEpAIBAAKCAQEAyHTEzLn5tXnpRdkUYLB9u5Pyax6fM60Nj4o8VmXl3ETZzGaF\r\n" \ + "B9X4J7BKNdBjngpuG7fa8H6r7gwQk4ZJGDTzqCrSV/Uu1C93KYRhTYJQj6eVSHD1\r\n" \ + "bk2y1RPD0hrt5kPqQhTrdOrA7R/UV06p86jt0uDBMHEwMjDV0/YI0FZPRo7yX/k9\r\n" \ + "Z5GIMC5Cst99++UMd//sMcB4j7/Cf8qtbCHWjdmLao5v4Jv4EFbMs44TFeY0BGbH\r\n" \ + "7vk2DmqV9gmaBmf0ZXH4yqSxJeD+PIs1BGe64E92hfx//DZrtenNLQNiTrM9AM+v\r\n" \ + "dqBpVoNq0qjU51Bx5rU2BXcFbXvI5MT9TNUhXwIDAQABAoIBAGdNtfYDiap6bzst\r\n" \ + "yhCiI8m9TtrhZw4MisaEaN/ll3XSjaOG2dvV6xMZCMV+5TeXDHOAZnY18Yi18vzz\r\n" \ + "4Ut2TnNFzizCECYNaA2fST3WgInnxUkV3YXAyP6CNxJaCmv2aA0yFr2kFVSeaKGt\r\n" \ + "ymvljNp2NVkvm7Th8fBQBO7I7AXhz43k0mR7XmPgewe8ApZOG3hstkOaMvbWAvWA\r\n" \ + "zCZupdDjZYjOJqlA4eEA4H8/w7F83r5CugeBE8LgEREjLPiyejrU5H1fubEY+h0d\r\n" \ + "l5HZBJ68ybTXfQ5U9o/QKA3dd0toBEhhdRUDGzWtjvwkEQfqF1reGWj/tod/gCpf\r\n" \ + "DFi6X0ECgYEA4wOv/pjSC3ty6TuOvKX2rOUiBrLXXv2JSxZnMoMiWI5ipLQt+RYT\r\n" \ + "VPafL/m7Dn6MbwjayOkcZhBwk5CNz5A6Q4lJ64Mq/lqHznRCQQ2Mc1G8eyDF/fYL\r\n" \ + "Ze2pLvwP9VD5jTc2miDfw+MnvJhywRRLcemDFP8k4hQVtm8PMp3ZmNECgYEA4gz7\r\n" \ + "wzObR4gn8ibe617uQPZjWzUj9dUHYd+in1gwBCIrtNnaRn9I9U/Q6tegRYpii4ys\r\n" \ + "c176NmU+umy6XmuSKV5qD9bSpZWG2nLFnslrN15Lm3fhZxoeMNhBaEDTnLT26yoi\r\n" \ + "33gp0mSSWy94ZEqipms+ULF6sY1ZtFW6tpGFoy8CgYAQHhnnvJflIs2ky4q10B60\r\n" \ + "ZcxFp3rtDpkp0JxhFLhiizFrujMtZSjYNm5U7KkgPVHhLELEUvCmOnKTt4ap/vZ0\r\n" \ + "BxJNe1GZH3pW6SAvGDQpl9sG7uu/vTFP+lCxukmzxB0DrrDcvorEkKMom7ZCCRvW\r\n" \ + "KZsZ6YeH2Z81BauRj218kQKBgQCUV/DgKP2985xDTT79N08jUo3hTP5MVYCCuj/+\r\n" \ + "UeEw1TvZcx3LJby7P6Xad6a1/BqveaGyFKIfEFIaBUBItk801sDDpDaYc4gL00Xc\r\n" \ + "7lFuBHOZkxJYlss5QrGpuOEl9ZwUt5IrFLBdYaKqNHzNVC1pCPfb/JyH6Dr2HUxq\r\n" \ + "gxUwAQKBgQCcU6G2L8AG9d9c0UpOyL1tMvFe5Ttw0KjlQVdsh1MP6yigYo9DYuwu\r\n" \ + "bHFVW2r0dBTqegP2/KTOxKzaHfC1qf0RGDsUoJCNJrd1cwoCLG8P2EF4w3OBrKqv\r\n" \ + "8u4ytY0F+Vlanj5lm3TaoHSVF1+NWPyOTiwevIECGKwSxvlki4fDAA==\r\n" \ + "-----END RSA PRIVATE KEY-----\r\n"/* END FILE */ + +/* This was generated from tests/data_files/cli-rsa.key.der using `xxd -i`. */ +/* BEGIN FILE binary macro TEST_CLI_KEY_RSA_DER tests/data_files/cli-rsa.key.der */ +#define TEST_CLI_KEY_RSA_DER { \ + 0x30, 0x82, 0x04, 0xa4, 0x02, 0x01, 0x00, 0x02, 0x82, 0x01, 0x01, 0x00, \ + 0xc8, 0x74, 0xc4, 0xcc, 0xb9, 0xf9, 0xb5, 0x79, 0xe9, 0x45, 0xd9, 0x14, \ + 0x60, 0xb0, 0x7d, 0xbb, 0x93, 0xf2, 0x6b, 0x1e, 0x9f, 0x33, 0xad, 0x0d, \ + 0x8f, 0x8a, 0x3c, 0x56, 0x65, 0xe5, 0xdc, 0x44, 0xd9, 0xcc, 0x66, 0x85, \ + 0x07, 0xd5, 0xf8, 0x27, 0xb0, 0x4a, 0x35, 0xd0, 0x63, 0x9e, 0x0a, 0x6e, \ + 0x1b, 0xb7, 0xda, 0xf0, 0x7e, 0xab, 0xee, 0x0c, 0x10, 0x93, 0x86, 0x49, \ + 0x18, 0x34, 0xf3, 0xa8, 0x2a, 0xd2, 0x57, 0xf5, 0x2e, 0xd4, 0x2f, 0x77, \ + 0x29, 0x84, 0x61, 0x4d, 0x82, 0x50, 0x8f, 0xa7, 0x95, 0x48, 0x70, 0xf5, \ + 0x6e, 0x4d, 0xb2, 0xd5, 0x13, 0xc3, 0xd2, 0x1a, 0xed, 0xe6, 0x43, 0xea, \ + 0x42, 0x14, 0xeb, 0x74, 0xea, 0xc0, 0xed, 0x1f, 0xd4, 0x57, 0x4e, 0xa9, \ + 0xf3, 0xa8, 0xed, 0xd2, 0xe0, 0xc1, 0x30, 0x71, 0x30, 0x32, 0x30, 0xd5, \ + 0xd3, 0xf6, 0x08, 0xd0, 0x56, 0x4f, 0x46, 0x8e, 0xf2, 0x5f, 0xf9, 0x3d, \ + 0x67, 0x91, 0x88, 0x30, 0x2e, 0x42, 0xb2, 0xdf, 0x7d, 0xfb, 0xe5, 0x0c, \ + 0x77, 0xff, 0xec, 0x31, 0xc0, 0x78, 0x8f, 0xbf, 0xc2, 0x7f, 0xca, 0xad, \ + 0x6c, 0x21, 0xd6, 0x8d, 0xd9, 0x8b, 0x6a, 0x8e, 0x6f, 0xe0, 0x9b, 0xf8, \ + 0x10, 0x56, 0xcc, 0xb3, 0x8e, 0x13, 0x15, 0xe6, 0x34, 0x04, 0x66, 0xc7, \ + 0xee, 0xf9, 0x36, 0x0e, 0x6a, 0x95, 0xf6, 0x09, 0x9a, 0x06, 0x67, 0xf4, \ + 0x65, 0x71, 0xf8, 0xca, 0xa4, 0xb1, 0x25, 0xe0, 0xfe, 0x3c, 0x8b, 0x35, \ + 0x04, 0x67, 0xba, 0xe0, 0x4f, 0x76, 0x85, 0xfc, 0x7f, 0xfc, 0x36, 0x6b, \ + 0xb5, 0xe9, 0xcd, 0x2d, 0x03, 0x62, 0x4e, 0xb3, 0x3d, 0x00, 0xcf, 0xaf, \ + 0x76, 0xa0, 0x69, 0x56, 0x83, 0x6a, 0xd2, 0xa8, 0xd4, 0xe7, 0x50, 0x71, \ + 0xe6, 0xb5, 0x36, 0x05, 0x77, 0x05, 0x6d, 0x7b, 0xc8, 0xe4, 0xc4, 0xfd, \ + 0x4c, 0xd5, 0x21, 0x5f, 0x02, 0x03, 0x01, 0x00, 0x01, 0x02, 0x82, 0x01, \ + 0x00, 0x67, 0x4d, 0xb5, 0xf6, 0x03, 0x89, 0xaa, 0x7a, 0x6f, 0x3b, 0x2d, \ + 0xca, 0x10, 0xa2, 0x23, 0xc9, 0xbd, 0x4e, 0xda, 0xe1, 0x67, 0x0e, 0x0c, \ + 0x8a, 0xc6, 0x84, 0x68, 0xdf, 0xe5, 0x97, 0x75, 0xd2, 0x8d, 0xa3, 0x86, \ + 0xd9, 0xdb, 0xd5, 0xeb, 0x13, 0x19, 0x08, 0xc5, 0x7e, 0xe5, 0x37, 0x97, \ + 0x0c, 0x73, 0x80, 0x66, 0x76, 0x35, 0xf1, 0x88, 0xb5, 0xf2, 0xfc, 0xf3, \ + 0xe1, 0x4b, 0x76, 0x4e, 0x73, 0x45, 0xce, 0x2c, 0xc2, 0x10, 0x26, 0x0d, \ + 0x68, 0x0d, 0x9f, 0x49, 0x3d, 0xd6, 0x80, 0x89, 0xe7, 0xc5, 0x49, 0x15, \ + 0xdd, 0x85, 0xc0, 0xc8, 0xfe, 0x82, 0x37, 0x12, 0x5a, 0x0a, 0x6b, 0xf6, \ + 0x68, 0x0d, 0x32, 0x16, 0xbd, 0xa4, 0x15, 0x54, 0x9e, 0x68, 0xa1, 0xad, \ + 0xca, 0x6b, 0xe5, 0x8c, 0xda, 0x76, 0x35, 0x59, 0x2f, 0x9b, 0xb4, 0xe1, \ + 0xf1, 0xf0, 0x50, 0x04, 0xee, 0xc8, 0xec, 0x05, 0xe1, 0xcf, 0x8d, 0xe4, \ + 0xd2, 0x64, 0x7b, 0x5e, 0x63, 0xe0, 0x7b, 0x07, 0xbc, 0x02, 0x96, 0x4e, \ + 0x1b, 0x78, 0x6c, 0xb6, 0x43, 0x9a, 0x32, 0xf6, 0xd6, 0x02, 0xf5, 0x80, \ + 0xcc, 0x26, 0x6e, 0xa5, 0xd0, 0xe3, 0x65, 0x88, 0xce, 0x26, 0xa9, 0x40, \ + 0xe1, 0xe1, 0x00, 0xe0, 0x7f, 0x3f, 0xc3, 0xb1, 0x7c, 0xde, 0xbe, 0x42, \ + 0xba, 0x07, 0x81, 0x13, 0xc2, 0xe0, 0x11, 0x11, 0x23, 0x2c, 0xf8, 0xb2, \ + 0x7a, 0x3a, 0xd4, 0xe4, 0x7d, 0x5f, 0xb9, 0xb1, 0x18, 0xfa, 0x1d, 0x1d, \ + 0x97, 0x91, 0xd9, 0x04, 0x9e, 0xbc, 0xc9, 0xb4, 0xd7, 0x7d, 0x0e, 0x54, \ + 0xf6, 0x8f, 0xd0, 0x28, 0x0d, 0xdd, 0x77, 0x4b, 0x68, 0x04, 0x48, 0x61, \ + 0x75, 0x15, 0x03, 0x1b, 0x35, 0xad, 0x8e, 0xfc, 0x24, 0x11, 0x07, 0xea, \ + 0x17, 0x5a, 0xde, 0x19, 0x68, 0xff, 0xb6, 0x87, 0x7f, 0x80, 0x2a, 0x5f, \ + 0x0c, 0x58, 0xba, 0x5f, 0x41, 0x02, 0x81, 0x81, 0x00, 0xe3, 0x03, 0xaf, \ + 0xfe, 0x98, 0xd2, 0x0b, 0x7b, 0x72, 0xe9, 0x3b, 0x8e, 0xbc, 0xa5, 0xf6, \ + 0xac, 0xe5, 0x22, 0x06, 0xb2, 0xd7, 0x5e, 0xfd, 0x89, 0x4b, 0x16, 0x67, \ + 0x32, 0x83, 0x22, 0x58, 0x8e, 0x62, 0xa4, 0xb4, 0x2d, 0xf9, 0x16, 0x13, \ + 0x54, 0xf6, 0x9f, 0x2f, 0xf9, 0xbb, 0x0e, 0x7e, 0x8c, 0x6f, 0x08, 0xda, \ + 0xc8, 0xe9, 0x1c, 0x66, 0x10, 0x70, 0x93, 0x90, 0x8d, 0xcf, 0x90, 0x3a, \ + 0x43, 0x89, 0x49, 0xeb, 0x83, 0x2a, 0xfe, 0x5a, 0x87, 0xce, 0x74, 0x42, \ + 0x41, 0x0d, 0x8c, 0x73, 0x51, 0xbc, 0x7b, 0x20, 0xc5, 0xfd, 0xf6, 0x0b, \ + 0x65, 0xed, 0xa9, 0x2e, 0xfc, 0x0f, 0xf5, 0x50, 0xf9, 0x8d, 0x37, 0x36, \ + 0x9a, 0x20, 0xdf, 0xc3, 0xe3, 0x27, 0xbc, 0x98, 0x72, 0xc1, 0x14, 0x4b, \ + 0x71, 0xe9, 0x83, 0x14, 0xff, 0x24, 0xe2, 0x14, 0x15, 0xb6, 0x6f, 0x0f, \ + 0x32, 0x9d, 0xd9, 0x98, 0xd1, 0x02, 0x81, 0x81, 0x00, 0xe2, 0x0c, 0xfb, \ + 0xc3, 0x33, 0x9b, 0x47, 0x88, 0x27, 0xf2, 0x26, 0xde, 0xeb, 0x5e, 0xee, \ + 0x40, 0xf6, 0x63, 0x5b, 0x35, 0x23, 0xf5, 0xd5, 0x07, 0x61, 0xdf, 0xa2, \ + 0x9f, 0x58, 0x30, 0x04, 0x22, 0x2b, 0xb4, 0xd9, 0xda, 0x46, 0x7f, 0x48, \ + 0xf5, 0x4f, 0xd0, 0xea, 0xd7, 0xa0, 0x45, 0x8a, 0x62, 0x8b, 0x8c, 0xac, \ + 0x73, 0x5e, 0xfa, 0x36, 0x65, 0x3e, 0xba, 0x6c, 0xba, 0x5e, 0x6b, 0x92, \ + 0x29, 0x5e, 0x6a, 0x0f, 0xd6, 0xd2, 0xa5, 0x95, 0x86, 0xda, 0x72, 0xc5, \ + 0x9e, 0xc9, 0x6b, 0x37, 0x5e, 0x4b, 0x9b, 0x77, 0xe1, 0x67, 0x1a, 0x1e, \ + 0x30, 0xd8, 0x41, 0x68, 0x40, 0xd3, 0x9c, 0xb4, 0xf6, 0xeb, 0x2a, 0x22, \ + 0xdf, 0x78, 0x29, 0xd2, 0x64, 0x92, 0x5b, 0x2f, 0x78, 0x64, 0x4a, 0xa2, \ + 0xa6, 0x6b, 0x3e, 0x50, 0xb1, 0x7a, 0xb1, 0x8d, 0x59, 0xb4, 0x55, 0xba, \ + 0xb6, 0x91, 0x85, 0xa3, 0x2f, 0x02, 0x81, 0x80, 0x10, 0x1e, 0x19, 0xe7, \ + 0xbc, 0x97, 0xe5, 0x22, 0xcd, 0xa4, 0xcb, 0x8a, 0xb5, 0xd0, 0x1e, 0xb4, \ + 0x65, 0xcc, 0x45, 0xa7, 0x7a, 0xed, 0x0e, 0x99, 0x29, 0xd0, 0x9c, 0x61, \ + 0x14, 0xb8, 0x62, 0x8b, 0x31, 0x6b, 0xba, 0x33, 0x2d, 0x65, 0x28, 0xd8, \ + 0x36, 0x6e, 0x54, 0xec, 0xa9, 0x20, 0x3d, 0x51, 0xe1, 0x2c, 0x42, 0xc4, \ + 0x52, 0xf0, 0xa6, 0x3a, 0x72, 0x93, 0xb7, 0x86, 0xa9, 0xfe, 0xf6, 0x74, \ + 0x07, 0x12, 0x4d, 0x7b, 0x51, 0x99, 0x1f, 0x7a, 0x56, 0xe9, 0x20, 0x2f, \ + 0x18, 0x34, 0x29, 0x97, 0xdb, 0x06, 0xee, 0xeb, 0xbf, 0xbd, 0x31, 0x4f, \ + 0xfa, 0x50, 0xb1, 0xba, 0x49, 0xb3, 0xc4, 0x1d, 0x03, 0xae, 0xb0, 0xdc, \ + 0xbe, 0x8a, 0xc4, 0x90, 0xa3, 0x28, 0x9b, 0xb6, 0x42, 0x09, 0x1b, 0xd6, \ + 0x29, 0x9b, 0x19, 0xe9, 0x87, 0x87, 0xd9, 0x9f, 0x35, 0x05, 0xab, 0x91, \ + 0x8f, 0x6d, 0x7c, 0x91, 0x02, 0x81, 0x81, 0x00, 0x94, 0x57, 0xf0, 0xe0, \ + 0x28, 0xfd, 0xbd, 0xf3, 0x9c, 0x43, 0x4d, 0x3e, 0xfd, 0x37, 0x4f, 0x23, \ + 0x52, 0x8d, 0xe1, 0x4c, 0xfe, 0x4c, 0x55, 0x80, 0x82, 0xba, 0x3f, 0xfe, \ + 0x51, 0xe1, 0x30, 0xd5, 0x3b, 0xd9, 0x73, 0x1d, 0xcb, 0x25, 0xbc, 0xbb, \ + 0x3f, 0xa5, 0xda, 0x77, 0xa6, 0xb5, 0xfc, 0x1a, 0xaf, 0x79, 0xa1, 0xb2, \ + 0x14, 0xa2, 0x1f, 0x10, 0x52, 0x1a, 0x05, 0x40, 0x48, 0xb6, 0x4f, 0x34, \ + 0xd6, 0xc0, 0xc3, 0xa4, 0x36, 0x98, 0x73, 0x88, 0x0b, 0xd3, 0x45, 0xdc, \ + 0xee, 0x51, 0x6e, 0x04, 0x73, 0x99, 0x93, 0x12, 0x58, 0x96, 0xcb, 0x39, \ + 0x42, 0xb1, 0xa9, 0xb8, 0xe1, 0x25, 0xf5, 0x9c, 0x14, 0xb7, 0x92, 0x2b, \ + 0x14, 0xb0, 0x5d, 0x61, 0xa2, 0xaa, 0x34, 0x7c, 0xcd, 0x54, 0x2d, 0x69, \ + 0x08, 0xf7, 0xdb, 0xfc, 0x9c, 0x87, 0xe8, 0x3a, 0xf6, 0x1d, 0x4c, 0x6a, \ + 0x83, 0x15, 0x30, 0x01, 0x02, 0x81, 0x81, 0x00, 0x9c, 0x53, 0xa1, 0xb6, \ + 0x2f, 0xc0, 0x06, 0xf5, 0xdf, 0x5c, 0xd1, 0x4a, 0x4e, 0xc8, 0xbd, 0x6d, \ + 0x32, 0xf1, 0x5e, 0xe5, 0x3b, 0x70, 0xd0, 0xa8, 0xe5, 0x41, 0x57, 0x6c, \ + 0x87, 0x53, 0x0f, 0xeb, 0x28, 0xa0, 0x62, 0x8f, 0x43, 0x62, 0xec, 0x2e, \ + 0x6c, 0x71, 0x55, 0x5b, 0x6a, 0xf4, 0x74, 0x14, 0xea, 0x7a, 0x03, 0xf6, \ + 0xfc, 0xa4, 0xce, 0xc4, 0xac, 0xda, 0x1d, 0xf0, 0xb5, 0xa9, 0xfd, 0x11, \ + 0x18, 0x3b, 0x14, 0xa0, 0x90, 0x8d, 0x26, 0xb7, 0x75, 0x73, 0x0a, 0x02, \ + 0x2c, 0x6f, 0x0f, 0xd8, 0x41, 0x78, 0xc3, 0x73, 0x81, 0xac, 0xaa, 0xaf, \ + 0xf2, 0xee, 0x32, 0xb5, 0x8d, 0x05, 0xf9, 0x59, 0x5a, 0x9e, 0x3e, 0x65, \ + 0x9b, 0x74, 0xda, 0xa0, 0x74, 0x95, 0x17, 0x5f, 0x8d, 0x58, 0xfc, 0x8e, \ + 0x4e, 0x2c, 0x1e, 0xbc, 0x81, 0x02, 0x18, 0xac, 0x12, 0xc6, 0xf9, 0x64, \ + 0x8b, 0x87, 0xc3, 0x00 \ +} +/* END FILE */ + +/* + * + * Test certificates and keys as C variables + * + */ + +/* + * CA + */ + +const char mbedtls_test_ca_crt_ec_pem[] = TEST_CA_CRT_EC_PEM; +const char mbedtls_test_ca_key_ec_pem[] = TEST_CA_KEY_EC_PEM; +const char mbedtls_test_ca_pwd_ec_pem[] = TEST_CA_PWD_EC_PEM; +const char mbedtls_test_ca_key_rsa_pem[] = TEST_CA_KEY_RSA_PEM; +const char mbedtls_test_ca_pwd_rsa_pem[] = TEST_CA_PWD_RSA_PEM; +const char mbedtls_test_ca_crt_rsa_sha1_pem[] = TEST_CA_CRT_RSA_SHA1_PEM; +const char mbedtls_test_ca_crt_rsa_sha256_pem[] = TEST_CA_CRT_RSA_SHA256_PEM; + +const unsigned char mbedtls_test_ca_crt_ec_der[] = TEST_CA_CRT_EC_DER; +const unsigned char mbedtls_test_ca_key_ec_der[] = TEST_CA_KEY_EC_DER; +const unsigned char mbedtls_test_ca_key_rsa_der[] = TEST_CA_KEY_RSA_DER; +const unsigned char mbedtls_test_ca_crt_rsa_sha1_der[] = + TEST_CA_CRT_RSA_SHA1_DER; +const unsigned char mbedtls_test_ca_crt_rsa_sha256_der[] = + TEST_CA_CRT_RSA_SHA256_DER; + +const size_t mbedtls_test_ca_crt_ec_pem_len = + sizeof( mbedtls_test_ca_crt_ec_pem ); +const size_t mbedtls_test_ca_key_ec_pem_len = + sizeof( mbedtls_test_ca_key_ec_pem ); +const size_t mbedtls_test_ca_pwd_ec_pem_len = + sizeof( mbedtls_test_ca_pwd_ec_pem ) - 1; +const size_t mbedtls_test_ca_key_rsa_pem_len = + sizeof( mbedtls_test_ca_key_rsa_pem ); +const size_t mbedtls_test_ca_pwd_rsa_pem_len = + sizeof( mbedtls_test_ca_pwd_rsa_pem ) - 1; +const size_t mbedtls_test_ca_crt_rsa_sha1_pem_len = + sizeof( mbedtls_test_ca_crt_rsa_sha1_pem ); +const size_t mbedtls_test_ca_crt_rsa_sha256_pem_len = + sizeof( mbedtls_test_ca_crt_rsa_sha256_pem ); + +const size_t mbedtls_test_ca_crt_ec_der_len = + sizeof( mbedtls_test_ca_crt_ec_der ); +const size_t mbedtls_test_ca_key_ec_der_len = + sizeof( mbedtls_test_ca_key_ec_der ); +const size_t mbedtls_test_ca_pwd_ec_der_len = 0; +const size_t mbedtls_test_ca_key_rsa_der_len = + sizeof( mbedtls_test_ca_key_rsa_der ); +const size_t mbedtls_test_ca_pwd_rsa_der_len = 0; +const size_t mbedtls_test_ca_crt_rsa_sha1_der_len = + sizeof( mbedtls_test_ca_crt_rsa_sha1_der ); +const size_t mbedtls_test_ca_crt_rsa_sha256_der_len = + sizeof( mbedtls_test_ca_crt_rsa_sha256_der ); + +/* + * Server + */ + +const char mbedtls_test_srv_crt_ec_pem[] = TEST_SRV_CRT_EC_PEM; +const char mbedtls_test_srv_key_ec_pem[] = TEST_SRV_KEY_EC_PEM; +const char mbedtls_test_srv_pwd_ec_pem[] = ""; +const char mbedtls_test_srv_key_rsa_pem[] = TEST_SRV_KEY_RSA_PEM; +const char mbedtls_test_srv_pwd_rsa_pem[] = ""; +const char mbedtls_test_srv_crt_rsa_sha1_pem[] = TEST_SRV_CRT_RSA_SHA1_PEM; +const char mbedtls_test_srv_crt_rsa_sha256_pem[] = TEST_SRV_CRT_RSA_SHA256_PEM; + +const unsigned char mbedtls_test_srv_crt_ec_der[] = TEST_SRV_CRT_EC_DER; +const unsigned char mbedtls_test_srv_key_ec_der[] = TEST_SRV_KEY_EC_DER; +const unsigned char mbedtls_test_srv_key_rsa_der[] = TEST_SRV_KEY_RSA_DER; +const unsigned char mbedtls_test_srv_crt_rsa_sha1_der[] = + TEST_SRV_CRT_RSA_SHA1_DER; +const unsigned char mbedtls_test_srv_crt_rsa_sha256_der[] = + TEST_SRV_CRT_RSA_SHA256_DER; + +const size_t mbedtls_test_srv_crt_ec_pem_len = + sizeof( mbedtls_test_srv_crt_ec_pem ); +const size_t mbedtls_test_srv_key_ec_pem_len = + sizeof( mbedtls_test_srv_key_ec_pem ); +const size_t mbedtls_test_srv_pwd_ec_pem_len = + sizeof( mbedtls_test_srv_pwd_ec_pem ) - 1; +const size_t mbedtls_test_srv_key_rsa_pem_len = + sizeof( mbedtls_test_srv_key_rsa_pem ); +const size_t mbedtls_test_srv_pwd_rsa_pem_len = + sizeof( mbedtls_test_srv_pwd_rsa_pem ) - 1; +const size_t mbedtls_test_srv_crt_rsa_sha1_pem_len = + sizeof( mbedtls_test_srv_crt_rsa_sha1_pem ); +const size_t mbedtls_test_srv_crt_rsa_sha256_pem_len = + sizeof( mbedtls_test_srv_crt_rsa_sha256_pem ); + +const size_t mbedtls_test_srv_crt_ec_der_len = + sizeof( mbedtls_test_srv_crt_ec_der ); +const size_t mbedtls_test_srv_key_ec_der_len = + sizeof( mbedtls_test_srv_key_ec_der ); +const size_t mbedtls_test_srv_pwd_ec_der_len = 0; +const size_t mbedtls_test_srv_key_rsa_der_len = + sizeof( mbedtls_test_srv_key_rsa_der ); +const size_t mbedtls_test_srv_pwd_rsa_der_len = 0; +const size_t mbedtls_test_srv_crt_rsa_sha1_der_len = + sizeof( mbedtls_test_srv_crt_rsa_sha1_der ); +const size_t mbedtls_test_srv_crt_rsa_sha256_der_len = + sizeof( mbedtls_test_srv_crt_rsa_sha256_der ); + +/* + * Client + */ + +const char mbedtls_test_cli_crt_ec_pem[] = TEST_CLI_CRT_EC_PEM; +const char mbedtls_test_cli_key_ec_pem[] = TEST_CLI_KEY_EC_PEM; +const char mbedtls_test_cli_pwd_ec_pem[] = ""; +const char mbedtls_test_cli_key_rsa_pem[] = TEST_CLI_KEY_RSA_PEM; +const char mbedtls_test_cli_pwd_rsa_pem[] = ""; +const char mbedtls_test_cli_crt_rsa_pem[] = TEST_CLI_CRT_RSA_PEM; + +const unsigned char mbedtls_test_cli_crt_ec_der[] = TEST_CLI_CRT_EC_DER; +const unsigned char mbedtls_test_cli_key_ec_der[] = TEST_CLI_KEY_EC_DER; +const unsigned char mbedtls_test_cli_key_rsa_der[] = TEST_CLI_KEY_RSA_DER; +const unsigned char mbedtls_test_cli_crt_rsa_der[] = TEST_CLI_CRT_RSA_DER; + +const size_t mbedtls_test_cli_crt_ec_pem_len = + sizeof( mbedtls_test_cli_crt_ec_pem ); +const size_t mbedtls_test_cli_key_ec_pem_len = + sizeof( mbedtls_test_cli_key_ec_pem ); +const size_t mbedtls_test_cli_pwd_ec_pem_len = + sizeof( mbedtls_test_cli_pwd_ec_pem ) - 1; +const size_t mbedtls_test_cli_key_rsa_pem_len = + sizeof( mbedtls_test_cli_key_rsa_pem ); +const size_t mbedtls_test_cli_pwd_rsa_pem_len = + sizeof( mbedtls_test_cli_pwd_rsa_pem ) - 1; +const size_t mbedtls_test_cli_crt_rsa_pem_len = + sizeof( mbedtls_test_cli_crt_rsa_pem ); + +const size_t mbedtls_test_cli_crt_ec_der_len = + sizeof( mbedtls_test_cli_crt_ec_der ); +const size_t mbedtls_test_cli_key_ec_der_len = + sizeof( mbedtls_test_cli_key_ec_der ); +const size_t mbedtls_test_cli_key_rsa_der_len = + sizeof( mbedtls_test_cli_key_rsa_der ); +const size_t mbedtls_test_cli_crt_rsa_der_len = + sizeof( mbedtls_test_cli_crt_rsa_der ); + +/* + * + * Definitions of test CRTs without specification of all parameters, choosing + * them automatically according to the config. For example, mbedtls_test_ca_crt + * is one of mbedtls_test_ca_crt_{rsa|ec}_{sha1|sha256}_{pem|der}. + * + */ + +/* + * Dispatch between PEM and DER according to config + */ + +#if defined(MBEDTLS_PEM_PARSE_C) + +/* PEM encoded test CA certificates and keys */ + +#define TEST_CA_KEY_RSA TEST_CA_KEY_RSA_PEM +#define TEST_CA_PWD_RSA TEST_CA_PWD_RSA_PEM +#define TEST_CA_CRT_RSA_SHA256 TEST_CA_CRT_RSA_SHA256_PEM +#define TEST_CA_CRT_RSA_SHA1 TEST_CA_CRT_RSA_SHA1_PEM +#define TEST_CA_KEY_EC TEST_CA_KEY_EC_PEM +#define TEST_CA_PWD_EC TEST_CA_PWD_EC_PEM +#define TEST_CA_CRT_EC TEST_CA_CRT_EC_PEM + +/* PEM encoded test server certificates and keys */ + +#define TEST_SRV_KEY_RSA TEST_SRV_KEY_RSA_PEM +#define TEST_SRV_PWD_RSA "" +#define TEST_SRV_CRT_RSA_SHA256 TEST_SRV_CRT_RSA_SHA256_PEM +#define TEST_SRV_CRT_RSA_SHA1 TEST_SRV_CRT_RSA_SHA1_PEM +#define TEST_SRV_KEY_EC TEST_SRV_KEY_EC_PEM +#define TEST_SRV_PWD_EC "" +#define TEST_SRV_CRT_EC TEST_SRV_CRT_EC_PEM + +/* PEM encoded test client certificates and keys */ + +#define TEST_CLI_KEY_RSA TEST_CLI_KEY_RSA_PEM +#define TEST_CLI_PWD_RSA "" +#define TEST_CLI_CRT_RSA TEST_CLI_CRT_RSA_PEM +#define TEST_CLI_KEY_EC TEST_CLI_KEY_EC_PEM +#define TEST_CLI_PWD_EC "" +#define TEST_CLI_CRT_EC TEST_CLI_CRT_EC_PEM + +#else /* MBEDTLS_PEM_PARSE_C */ + +/* DER encoded test CA certificates and keys */ + +#define TEST_CA_KEY_RSA TEST_CA_KEY_RSA_DER +#define TEST_CA_PWD_RSA "" +#define TEST_CA_CRT_RSA_SHA256 TEST_CA_CRT_RSA_SHA256_DER +#define TEST_CA_CRT_RSA_SHA1 TEST_CA_CRT_RSA_SHA1_DER +#define TEST_CA_KEY_EC TEST_CA_KEY_EC_DER +#define TEST_CA_PWD_EC "" +#define TEST_CA_CRT_EC TEST_CA_CRT_EC_DER + +/* DER encoded test server certificates and keys */ + +#define TEST_SRV_KEY_RSA TEST_SRV_KEY_RSA_DER +#define TEST_SRV_PWD_RSA "" +#define TEST_SRV_CRT_RSA_SHA256 TEST_SRV_CRT_RSA_SHA256_DER +#define TEST_SRV_CRT_RSA_SHA1 TEST_SRV_CRT_RSA_SHA1_DER +#define TEST_SRV_KEY_EC TEST_SRV_KEY_EC_DER +#define TEST_SRV_PWD_EC "" +#define TEST_SRV_CRT_EC TEST_SRV_CRT_EC_DER + +/* DER encoded test client certificates and keys */ + +#define TEST_CLI_KEY_RSA TEST_CLI_KEY_RSA_DER +#define TEST_CLI_PWD_RSA "" +#define TEST_CLI_CRT_RSA TEST_CLI_CRT_RSA_DER +#define TEST_CLI_KEY_EC TEST_CLI_KEY_EC_DER +#define TEST_CLI_PWD_EC "" +#define TEST_CLI_CRT_EC TEST_CLI_CRT_EC_DER + +#endif /* MBEDTLS_PEM_PARSE_C */ + +const char mbedtls_test_ca_key_rsa[] = TEST_CA_KEY_RSA; +const char mbedtls_test_ca_pwd_rsa[] = TEST_CA_PWD_RSA; +const char mbedtls_test_ca_crt_rsa_sha256[] = TEST_CA_CRT_RSA_SHA256; +const char mbedtls_test_ca_crt_rsa_sha1[] = TEST_CA_CRT_RSA_SHA1; +const char mbedtls_test_ca_key_ec[] = TEST_CA_KEY_EC; +const char mbedtls_test_ca_pwd_ec[] = TEST_CA_PWD_EC; +const char mbedtls_test_ca_crt_ec[] = TEST_CA_CRT_EC; + +const char mbedtls_test_srv_key_rsa[] = TEST_SRV_KEY_RSA; +const char mbedtls_test_srv_pwd_rsa[] = TEST_SRV_PWD_RSA; +const char mbedtls_test_srv_crt_rsa_sha256[] = TEST_SRV_CRT_RSA_SHA256; +const char mbedtls_test_srv_crt_rsa_sha1[] = TEST_SRV_CRT_RSA_SHA1; +const char mbedtls_test_srv_key_ec[] = TEST_SRV_KEY_EC; +const char mbedtls_test_srv_pwd_ec[] = TEST_SRV_PWD_EC; +const char mbedtls_test_srv_crt_ec[] = TEST_SRV_CRT_EC; + +const char mbedtls_test_cli_key_rsa[] = TEST_CLI_KEY_RSA; +const char mbedtls_test_cli_pwd_rsa[] = TEST_CLI_PWD_RSA; +const char mbedtls_test_cli_crt_rsa[] = TEST_CLI_CRT_RSA; +const char mbedtls_test_cli_key_ec[] = TEST_CLI_KEY_EC; +const char mbedtls_test_cli_pwd_ec[] = TEST_CLI_PWD_EC; +const char mbedtls_test_cli_crt_ec[] = TEST_CLI_CRT_EC; + +const size_t mbedtls_test_ca_key_rsa_len = + sizeof( mbedtls_test_ca_key_rsa ); +const size_t mbedtls_test_ca_pwd_rsa_len = + sizeof( mbedtls_test_ca_pwd_rsa ) - 1; +const size_t mbedtls_test_ca_crt_rsa_sha256_len = + sizeof( mbedtls_test_ca_crt_rsa_sha256 ); +const size_t mbedtls_test_ca_crt_rsa_sha1_len = + sizeof( mbedtls_test_ca_crt_rsa_sha1 ); +const size_t mbedtls_test_ca_key_ec_len = + sizeof( mbedtls_test_ca_key_ec ); +const size_t mbedtls_test_ca_pwd_ec_len = + sizeof( mbedtls_test_ca_pwd_ec ) - 1; +const size_t mbedtls_test_ca_crt_ec_len = + sizeof( mbedtls_test_ca_crt_ec ); + +const size_t mbedtls_test_srv_key_rsa_len = + sizeof( mbedtls_test_srv_key_rsa ); +const size_t mbedtls_test_srv_pwd_rsa_len = + sizeof( mbedtls_test_srv_pwd_rsa ) -1; +const size_t mbedtls_test_srv_crt_rsa_sha256_len = + sizeof( mbedtls_test_srv_crt_rsa_sha256 ); +const size_t mbedtls_test_srv_crt_rsa_sha1_len = + sizeof( mbedtls_test_srv_crt_rsa_sha1 ); +const size_t mbedtls_test_srv_key_ec_len = + sizeof( mbedtls_test_srv_key_ec ); +const size_t mbedtls_test_srv_pwd_ec_len = + sizeof( mbedtls_test_srv_pwd_ec ) - 1; +const size_t mbedtls_test_srv_crt_ec_len = + sizeof( mbedtls_test_srv_crt_ec ); + +const size_t mbedtls_test_cli_key_rsa_len = + sizeof( mbedtls_test_cli_key_rsa ); +const size_t mbedtls_test_cli_pwd_rsa_len = + sizeof( mbedtls_test_cli_pwd_rsa ) - 1; +const size_t mbedtls_test_cli_crt_rsa_len = + sizeof( mbedtls_test_cli_crt_rsa ); +const size_t mbedtls_test_cli_key_ec_len = + sizeof( mbedtls_test_cli_key_ec ); +const size_t mbedtls_test_cli_pwd_ec_len = + sizeof( mbedtls_test_cli_pwd_ec ) - 1; +const size_t mbedtls_test_cli_crt_ec_len = + sizeof( mbedtls_test_cli_crt_ec ); + +/* + * Dispatch between SHA-1 and SHA-256 + */ + +#if defined(MBEDTLS_SHA256_C) +#define TEST_CA_CRT_RSA TEST_CA_CRT_RSA_SHA256 +#define TEST_SRV_CRT_RSA TEST_SRV_CRT_RSA_SHA256 +#else +#define TEST_CA_CRT_RSA TEST_CA_CRT_RSA_SHA1 +#define TEST_SRV_CRT_RSA TEST_SRV_CRT_RSA_SHA1 +#endif /* MBEDTLS_SHA256_C */ + +const char mbedtls_test_ca_crt_rsa[] = TEST_CA_CRT_RSA; +const char mbedtls_test_srv_crt_rsa[] = TEST_SRV_CRT_RSA; + +const size_t mbedtls_test_ca_crt_rsa_len = + sizeof( mbedtls_test_ca_crt_rsa ); +const size_t mbedtls_test_srv_crt_rsa_len = + sizeof( mbedtls_test_srv_crt_rsa ); + +/* + * Dispatch between RSA and EC + */ #if defined(MBEDTLS_RSA_C) -#if defined(MBEDTLS_SHA256_C) -#define TEST_CA_CRT_RSA_SHA256 \ -"-----BEGIN CERTIFICATE-----\r\n" \ -"MIIDhzCCAm+gAwIBAgIBADANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ -"MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ -"MTcwNTA0MTY1NzAxWhcNMjcwNTA1MTY1NzAxWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ -"A1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ -"CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ -"mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ -"50BwFMtEonILwuVA+T7lpg6z+exKY8C4KQB0nFc7qKUEkHHxvYPZP9al4jwqj+8n\r\n" \ -"YMPGn8u67GB9t+aEMr5P+1gmIgNb1LTV+/Xjli5wwOQuvfwu7uJBVcA0Ln0kcmnL\r\n" \ -"R7EUQIN9Z/SG9jGr8XmksrUuEvmEF/Bibyc+E1ixVA0hmnM3oTDPb5Lc9un8rNsu\r\n" \ -"KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ -"gZUwgZIwHQYDVR0OBBYEFLRa5KWz3tJS9rnVppUP6z68x/3/MGMGA1UdIwRcMFqA\r\n" \ -"FLRa5KWz3tJS9rnVppUP6z68x/3/oT+kPTA7MQswCQYDVQQGEwJOTDERMA8GA1UE\r\n" \ -"CgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0GCAQAwDAYDVR0T\r\n" \ -"BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAHK/HHrTZMnnVMpde1io+voAtql7j\r\n" \ -"4sRhLrjD7o3THtwRbDa2diCvpq0Sq23Ng2LMYoXsOxoL/RQK3iN7UKxV3MKPEr0w\r\n" \ -"XQS+kKQqiT2bsfrjnWMVHZtUOMpm6FNqcdGm/Rss3vKda2lcKl8kUnq/ylc1+QbB\r\n" \ -"G6A6tUvQcr2ZyWfVg+mM5XkhTrOOXus2OLikb4WwEtJTJRNE0f+yPODSUz0/vT57\r\n" \ -"ApH0CnB80bYJshYHPHHymOtleAB8KSYtqm75g/YNobjnjB6cm4HkW3OZRVIl6fYY\r\n" \ -"n20NRVA1Vjs6GAROr4NqW4k/+LofY9y0LLDE+p0oIEKXIsIvhPr39swxSA==\r\n" \ -"-----END CERTIFICATE-----\r\n" +#define TEST_CA_KEY TEST_CA_KEY_RSA +#define TEST_CA_PWD TEST_CA_PWD_RSA +#define TEST_CA_CRT TEST_CA_CRT_RSA -const char mbedtls_test_ca_crt_rsa[] = TEST_CA_CRT_RSA_SHA256; -const size_t mbedtls_test_ca_crt_rsa_len = sizeof( mbedtls_test_ca_crt_rsa ); -#define TEST_CA_CRT_RSA_SOME +#define TEST_SRV_KEY TEST_SRV_KEY_RSA +#define TEST_SRV_PWD TEST_SRV_PWD_RSA +#define TEST_SRV_CRT TEST_SRV_CRT_RSA -static const char mbedtls_test_ca_crt_rsa_sha256[] = TEST_CA_CRT_RSA_SHA256; +#define TEST_CLI_KEY TEST_CLI_KEY_RSA +#define TEST_CLI_PWD TEST_CLI_PWD_RSA +#define TEST_CLI_CRT TEST_CLI_CRT_RSA -#endif +#else /* no RSA, so assume ECDSA */ -#if !defined(TEST_CA_CRT_RSA_SOME) || defined(MBEDTLS_SHA1_C) -#define TEST_CA_CRT_RSA_SHA1 \ -"-----BEGIN CERTIFICATE-----\r\n" \ -"MIIDhzCCAm+gAwIBAgIBADANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" \ -"MA8GA1UEChMIUG9sYXJTU0wxGTAXBgNVBAMTEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ -"MTEwMjEyMTQ0NDAwWhcNMjEwMjEyMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ -"A1UEChMIUG9sYXJTU0wxGTAXBgNVBAMTEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ -"CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ -"mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ -"50BwFMtEonILwuVA+T7lpg6z+exKY8C4KQB0nFc7qKUEkHHxvYPZP9al4jwqj+8n\r\n" \ -"YMPGn8u67GB9t+aEMr5P+1gmIgNb1LTV+/Xjli5wwOQuvfwu7uJBVcA0Ln0kcmnL\r\n" \ -"R7EUQIN9Z/SG9jGr8XmksrUuEvmEF/Bibyc+E1ixVA0hmnM3oTDPb5Lc9un8rNsu\r\n" \ -"KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ -"gZUwgZIwDAYDVR0TBAUwAwEB/zAdBgNVHQ4EFgQUtFrkpbPe0lL2udWmlQ/rPrzH\r\n" \ -"/f8wYwYDVR0jBFwwWoAUtFrkpbPe0lL2udWmlQ/rPrzH/f+hP6Q9MDsxCzAJBgNV\r\n" \ -"BAYTAk5MMREwDwYDVQQKEwhQb2xhclNTTDEZMBcGA1UEAxMQUG9sYXJTU0wgVGVz\r\n" \ -"dCBDQYIBADANBgkqhkiG9w0BAQUFAAOCAQEAuP1U2ABUkIslsCfdlc2i94QHHYeJ\r\n" \ -"SsR4EdgHtdciUI5I62J6Mom+Y0dT/7a+8S6MVMCZP6C5NyNyXw1GWY/YR82XTJ8H\r\n" \ -"DBJiCTok5DbZ6SzaONBzdWHXwWwmi5vg1dxn7YxrM9d0IjxM27WNKs4sDQhZBQkF\r\n" \ -"pjmfs2cb4oPl4Y9T9meTx/lvdkRYEug61Jfn6cA+qHpyPYdTH+UshITnmp5/Ztkf\r\n" \ -"m/UTSLBNFNHesiTZeH31NcxYGdHSme9Nc/gfidRa0FLOCfWxRlFqAI47zG9jAQCZ\r\n" \ -"7Z2mCGDNMhjQc+BYcdnl0lPXjdDK6V0qCg1dVewhUBcW5gZKzV7e9+DpVA==\r\n" \ -"-----END CERTIFICATE-----\r\n" +#define TEST_CA_KEY TEST_CA_KEY_EC +#define TEST_CA_PWD TEST_CA_PWD_EC +#define TEST_CA_CRT TEST_CA_CRT_EC -#if !defined (TEST_CA_CRT_RSA_SOME) -const char mbedtls_test_ca_crt_rsa[] = TEST_CA_CRT_RSA_SHA1; -const size_t mbedtls_test_ca_crt_rsa_len = sizeof( mbedtls_test_ca_crt_rsa ); -#endif +#define TEST_SRV_KEY TEST_SRV_KEY_EC +#define TEST_SRV_PWD TEST_SRV_PWD_EC +#define TEST_SRV_CRT TEST_SRV_CRT_EC -static const char mbedtls_test_ca_crt_rsa_sha1[] = TEST_CA_CRT_RSA_SHA1; +#define TEST_CLI_KEY TEST_CLI_KEY_EC +#define TEST_CLI_PWD TEST_CLI_PWD_EC +#define TEST_CLI_CRT TEST_CLI_CRT_EC -#endif - -const char mbedtls_test_ca_key_rsa[] = -"-----BEGIN RSA PRIVATE KEY-----\r\n" -"Proc-Type: 4,ENCRYPTED\r\n" -"DEK-Info: DES-EDE3-CBC,A8A95B05D5B7206B\r\n" -"\r\n" -"9Qd9GeArejl1GDVh2lLV1bHt0cPtfbh5h/5zVpAVaFpqtSPMrElp50Rntn9et+JA\r\n" -"7VOyboR+Iy2t/HU4WvA687k3Bppe9GwKHjHhtl//8xFKwZr3Xb5yO5JUP8AUctQq\r\n" -"Nb8CLlZyuUC+52REAAthdWgsX+7dJO4yabzUcQ22Tp9JSD0hiL43BlkWYUNK3dAo\r\n" -"PZlmiptjnzVTjg1MxsBSydZinWOLBV8/JQgxSPo2yD4uEfig28qbvQ2wNIn0pnAb\r\n" -"GxnSAOazkongEGfvcjIIs+LZN9gXFhxcOh6kc4Q/c99B7QWETwLLkYgZ+z1a9VY9\r\n" -"gEU7CwCxYCD+h9hY6FPmsK0/lC4O7aeRKpYq00rPPxs6i7phiexg6ax6yTMmArQq\r\n" -"QmK3TAsJm8V/J5AWpLEV6jAFgRGymGGHnof0DXzVWZidrcZJWTNuGEX90nB3ee2w\r\n" -"PXJEFWKoD3K3aFcSLdHYr3mLGxP7H9ThQai9VsycxZKS5kwvBKQ//YMrmFfwPk8x\r\n" -"vTeY4KZMaUrveEel5tWZC94RSMKgxR6cyE1nBXyTQnDOGbfpNNgBKxyKbINWoOJU\r\n" -"WJZAwlsQn+QzCDwpri7+sV1mS3gBE6UY7aQmnmiiaC2V3Hbphxct/en5QsfDOt1X\r\n" -"JczSfpRWLlbPznZg8OQh/VgCMA58N5DjOzTIK7sJJ5r+94ZBTCpgAMbF588f0NTR\r\n" -"KCe4yrxGJR7X02M4nvD4IwOlpsQ8xQxZtOSgXv4LkxvdU9XJJKWZ/XNKJeWztxSe\r\n" -"Z1vdTc2YfsDBA2SEv33vxHx2g1vqtw8SjDRT2RaQSS0QuSaMJimdOX6mTOCBKk1J\r\n" -"9Q5mXTrER+/LnK0jEmXsBXWA5bqqVZIyahXSx4VYZ7l7w/PHiUDtDgyRhMMKi4n2\r\n" -"iQvQcWSQTjrpnlJbca1/DkpRt3YwrvJwdqb8asZU2VrNETh5x0QVefDRLFiVpif/\r\n" -"tUaeAe/P1F8OkS7OIZDs1SUbv/sD2vMbhNkUoCms3/PvNtdnvgL4F0zhaDpKCmlT\r\n" -"P8vx49E7v5CyRNmED9zZg4o3wmMqrQO93PtTug3Eu9oVx1zPQM1NVMyBa2+f29DL\r\n" -"1nuTCeXdo9+ni45xx+jAI4DCwrRdhJ9uzZyC6962H37H6D+5naNvClFR1s6li1Gb\r\n" -"nqPoiy/OBsEx9CaDGcqQBp5Wme/3XW+6z1ISOx+igwNTVCT14mHdBMbya0eIKft5\r\n" -"X+GnwtgEMyCYyyWuUct8g4RzErcY9+yW9Om5Hzpx4zOuW4NPZgPDTgK+t2RSL/Yq\r\n" -"rE1njrgeGYcVeG3f+OftH4s6fPbq7t1A5ZgUscbLMBqr9tK+OqygR4EgKBPsH6Cz\r\n" -"L6zlv/2RV0qAHvVuDJcIDIgwY5rJtINEm32rhOeFNJwZS5MNIC1czXZx5//ugX7l\r\n" -"I4sy5nbVhwSjtAk8Xg5dZbdTZ6mIrb7xqH+fdakZor1khG7bC2uIwibD3cSl2XkR\r\n" -"wN48lslbHnqqagr6Xm1nNOSVl8C/6kbJEsMpLhAezfRtGwvOucoaE+WbeUNolGde\r\n" -"P/eQiddSf0brnpiLJRh7qZrl9XuqYdpUqnoEdMAfotDOID8OtV7gt8a48ad8VPW2\r\n" -"-----END RSA PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_ca_key_rsa_len = sizeof( mbedtls_test_ca_key_rsa ); - -const char mbedtls_test_ca_pwd_rsa[] = "PolarSSLTest"; -const size_t mbedtls_test_ca_pwd_rsa_len = sizeof( mbedtls_test_ca_pwd_rsa ) - 1; - -/* tests/data_files/server2.crt */ -const char mbedtls_test_srv_crt_rsa[] = -"-----BEGIN CERTIFICATE-----\r\n" -"MIIDNzCCAh+gAwIBAgIBAjANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" -"MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" -"MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UECgwIUG9sYXJTU0wxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN\r\n" -"AQEBBQADggEPADCCAQoCggEBAMFNo93nzR3RBNdJcriZrA545Do8Ss86ExbQWuTN\r\n" -"owCIp+4ea5anUrSQ7y1yej4kmvy2NKwk9XfgJmSMnLAofaHa6ozmyRyWvP7BBFKz\r\n" -"NtSj+uGxdtiQwWG0ZlI2oiZTqqt0Xgd9GYLbKtgfoNkNHC1JZvdbJXNG6AuKT2kM\r\n" -"tQCQ4dqCEGZ9rlQri2V5kaHiYcPNQEkI7mgM8YuG0ka/0LiqEQMef1aoGh5EGA8P\r\n" -"hYvai0Re4hjGYi/HZo36Xdh98yeJKQHFkA4/J/EwyEoO79bex8cna8cFPXrEAjya\r\n" -"HT4P6DSYW8tzS1KW2BGiLICIaTla0w+w3lkvEcf36hIBMJcCAwEAAaNNMEswCQYD\r\n" -"VR0TBAIwADAdBgNVHQ4EFgQUpQXoZLjc32APUBJNYKhkr02LQ5MwHwYDVR0jBBgw\r\n" -"FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQEFBQADggEBAAFzC0rF\r\n" -"y6De8WMcdgQrEw3AhBHFjzqnxZw1ene4IBSC7lTw8rBSy3jOWQdPUWn+0y/pCeeF\r\n" -"kti6sevFdl1hLemGtd4q+T9TKEKGg3ND4ARfB5AUZZ9uEHq8WBkiwus5clGS17Qd\r\n" -"dS/TOisB59tQruLx1E1bPLtBKyqk4koC5WAULJwfpswGSyWJTpYwIpxcWE3D2tBu\r\n" -"UB6MZfXZFzWmWEOyKbeoXjXe8GBCGgHLywvYDsGQ36HSGtEsAvR2QaTLSxWYcfk1\r\n" -"fbDn4jSWkb4yZy1r01UEigFQtONieGwRFaUqEcFJHJvEEGVgh9keaVlOj2vrwf5r\r\n" -"4mN4lW7gLdenN6g=\r\n" -"-----END CERTIFICATE-----\r\n"; -const size_t mbedtls_test_srv_crt_rsa_len = sizeof( mbedtls_test_srv_crt_rsa ); - -/* tests/data_files/server2.key */ -const char mbedtls_test_srv_key_rsa[] = -"-----BEGIN RSA PRIVATE KEY-----\r\n" -"MIIEpAIBAAKCAQEAwU2j3efNHdEE10lyuJmsDnjkOjxKzzoTFtBa5M2jAIin7h5r\r\n" -"lqdStJDvLXJ6PiSa/LY0rCT1d+AmZIycsCh9odrqjObJHJa8/sEEUrM21KP64bF2\r\n" -"2JDBYbRmUjaiJlOqq3ReB30Zgtsq2B+g2Q0cLUlm91slc0boC4pPaQy1AJDh2oIQ\r\n" -"Zn2uVCuLZXmRoeJhw81ASQjuaAzxi4bSRr/QuKoRAx5/VqgaHkQYDw+Fi9qLRF7i\r\n" -"GMZiL8dmjfpd2H3zJ4kpAcWQDj8n8TDISg7v1t7HxydrxwU9esQCPJodPg/oNJhb\r\n" -"y3NLUpbYEaIsgIhpOVrTD7DeWS8Rx/fqEgEwlwIDAQABAoIBAQCXR0S8EIHFGORZ\r\n" -"++AtOg6eENxD+xVs0f1IeGz57Tjo3QnXX7VBZNdj+p1ECvhCE/G7XnkgU5hLZX+G\r\n" -"Z0jkz/tqJOI0vRSdLBbipHnWouyBQ4e/A1yIJdlBtqXxJ1KE/ituHRbNc4j4kL8Z\r\n" -"/r6pvwnTI0PSx2Eqs048YdS92LT6qAv4flbNDxMn2uY7s4ycS4Q8w1JXnCeaAnYm\r\n" -"WYI5wxO+bvRELR2Mcz5DmVnL8jRyml6l6582bSv5oufReFIbyPZbQWlXgYnpu6He\r\n" -"GTc7E1zKYQGG/9+DQUl/1vQuCPqQwny0tQoX2w5tdYpdMdVm+zkLtbajzdTviJJa\r\n" -"TWzL6lt5AoGBAN86+SVeJDcmQJcv4Eq6UhtRr4QGMiQMz0Sod6ettYxYzMgxtw28\r\n" -"CIrgpozCc+UaZJLo7UxvC6an85r1b2nKPCLQFaggJ0H4Q0J/sZOhBIXaoBzWxveK\r\n" -"nupceKdVxGsFi8CDy86DBfiyFivfBj+47BbaQzPBj7C4rK7UlLjab2rDAoGBAN2u\r\n" -"AM2gchoFiu4v1HFL8D7lweEpi6ZnMJjnEu/dEgGQJFjwdpLnPbsj4c75odQ4Gz8g\r\n" -"sw9lao9VVzbusoRE/JGI4aTdO0pATXyG7eG1Qu+5Yc1YGXcCrliA2xM9xx+d7f+s\r\n" -"mPzN+WIEg5GJDYZDjAzHG5BNvi/FfM1C9dOtjv2dAoGAF0t5KmwbjWHBhcVqO4Ic\r\n" -"BVvN3BIlc1ue2YRXEDlxY5b0r8N4XceMgKmW18OHApZxfl8uPDauWZLXOgl4uepv\r\n" -"whZC3EuWrSyyICNhLY21Ah7hbIEBPF3L3ZsOwC+UErL+dXWLdB56Jgy3gZaBeW7b\r\n" -"vDrEnocJbqCm7IukhXHOBK8CgYEAwqdHB0hqyNSzIOGY7v9abzB6pUdA3BZiQvEs\r\n" -"3LjHVd4HPJ2x0N8CgrBIWOE0q8+0hSMmeE96WW/7jD3fPWwCR5zlXknxBQsfv0gP\r\n" -"3BC5PR0Qdypz+d+9zfMf625kyit4T/hzwhDveZUzHnk1Cf+IG7Q+TOEnLnWAWBED\r\n" -"ISOWmrUCgYAFEmRxgwAc/u+D6t0syCwAYh6POtscq9Y0i9GyWk89NzgC4NdwwbBH\r\n" -"4AgahOxIxXx2gxJnq3yfkJfIjwf0s2DyP0kY2y6Ua1OeomPeY9mrIS4tCuDQ6LrE\r\n" -"TB6l9VGoxJL4fyHnZb8L5gGvnB1bbD8cL6YPaDiOhcRseC9vBiEuVg==\r\n" -"-----END RSA PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_srv_key_rsa_len = sizeof( mbedtls_test_srv_key_rsa ); - -/* tests/data_files/cli-rsa-sha256.crt */ -const char mbedtls_test_cli_crt_rsa[] = -"-----BEGIN CERTIFICATE-----\r\n" -"MIIDPzCCAiegAwIBAgIBBDANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" -"MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" -"MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA8MQswCQYDVQQGEwJOTDERMA8G\r\n" -"A1UECgwIUG9sYXJTU0wxGjAYBgNVBAMMEVBvbGFyU1NMIENsaWVudCAyMIIBIjAN\r\n" -"BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyHTEzLn5tXnpRdkUYLB9u5Pyax6f\r\n" -"M60Nj4o8VmXl3ETZzGaFB9X4J7BKNdBjngpuG7fa8H6r7gwQk4ZJGDTzqCrSV/Uu\r\n" -"1C93KYRhTYJQj6eVSHD1bk2y1RPD0hrt5kPqQhTrdOrA7R/UV06p86jt0uDBMHEw\r\n" -"MjDV0/YI0FZPRo7yX/k9Z5GIMC5Cst99++UMd//sMcB4j7/Cf8qtbCHWjdmLao5v\r\n" -"4Jv4EFbMs44TFeY0BGbH7vk2DmqV9gmaBmf0ZXH4yqSxJeD+PIs1BGe64E92hfx/\r\n" -"/DZrtenNLQNiTrM9AM+vdqBpVoNq0qjU51Bx5rU2BXcFbXvI5MT9TNUhXwIDAQAB\r\n" -"o00wSzAJBgNVHRMEAjAAMB0GA1UdDgQWBBRxoQBzckAvVHZeM/xSj7zx3WtGITAf\r\n" -"BgNVHSMEGDAWgBS0WuSls97SUva51aaVD+s+vMf9/zANBgkqhkiG9w0BAQsFAAOC\r\n" -"AQEAlHabem2Tu69VUN7EipwnQn1dIHdgvT5i+iQHpSxY1crPnBbAeSdAXwsVEqLQ\r\n" -"gOOIAQD5VIITNuoGgo4i+4OpNh9u7ZkpRHla+/swsfrFWRRbBNP5Bcu74AGLstwU\r\n" -"zM8gIkBiyfM1Q1qDQISV9trlCG6O8vh8dp/rbI3rfzo99BOHXgFCrzXjCuW4vDsF\r\n" -"r+Dao26bX3sJ6UnEWg1H3o2x6PpUcvQ36h71/bz4TEbbUUEpe02V4QWuL+wrhHJL\r\n" -"U7o3SVE3Og7jPF8sat0a50YUWhwEFI256m02KAXLg89ueUyYKEr6rNwhcvXJpvU9\r\n" -"giIVvd0Sbjjnn7NC4VDbcXV8vw==\r\n" -"-----END CERTIFICATE-----\r\n"; -const size_t mbedtls_test_cli_crt_rsa_len = sizeof( mbedtls_test_cli_crt_rsa ); - -/* tests/data_files/cli-rsa.key */ -const char mbedtls_test_cli_key_rsa[] = -"-----BEGIN RSA PRIVATE KEY-----\r\n" -"MIIEpAIBAAKCAQEAyHTEzLn5tXnpRdkUYLB9u5Pyax6fM60Nj4o8VmXl3ETZzGaF\r\n" -"B9X4J7BKNdBjngpuG7fa8H6r7gwQk4ZJGDTzqCrSV/Uu1C93KYRhTYJQj6eVSHD1\r\n" -"bk2y1RPD0hrt5kPqQhTrdOrA7R/UV06p86jt0uDBMHEwMjDV0/YI0FZPRo7yX/k9\r\n" -"Z5GIMC5Cst99++UMd//sMcB4j7/Cf8qtbCHWjdmLao5v4Jv4EFbMs44TFeY0BGbH\r\n" -"7vk2DmqV9gmaBmf0ZXH4yqSxJeD+PIs1BGe64E92hfx//DZrtenNLQNiTrM9AM+v\r\n" -"dqBpVoNq0qjU51Bx5rU2BXcFbXvI5MT9TNUhXwIDAQABAoIBAGdNtfYDiap6bzst\r\n" -"yhCiI8m9TtrhZw4MisaEaN/ll3XSjaOG2dvV6xMZCMV+5TeXDHOAZnY18Yi18vzz\r\n" -"4Ut2TnNFzizCECYNaA2fST3WgInnxUkV3YXAyP6CNxJaCmv2aA0yFr2kFVSeaKGt\r\n" -"ymvljNp2NVkvm7Th8fBQBO7I7AXhz43k0mR7XmPgewe8ApZOG3hstkOaMvbWAvWA\r\n" -"zCZupdDjZYjOJqlA4eEA4H8/w7F83r5CugeBE8LgEREjLPiyejrU5H1fubEY+h0d\r\n" -"l5HZBJ68ybTXfQ5U9o/QKA3dd0toBEhhdRUDGzWtjvwkEQfqF1reGWj/tod/gCpf\r\n" -"DFi6X0ECgYEA4wOv/pjSC3ty6TuOvKX2rOUiBrLXXv2JSxZnMoMiWI5ipLQt+RYT\r\n" -"VPafL/m7Dn6MbwjayOkcZhBwk5CNz5A6Q4lJ64Mq/lqHznRCQQ2Mc1G8eyDF/fYL\r\n" -"Ze2pLvwP9VD5jTc2miDfw+MnvJhywRRLcemDFP8k4hQVtm8PMp3ZmNECgYEA4gz7\r\n" -"wzObR4gn8ibe617uQPZjWzUj9dUHYd+in1gwBCIrtNnaRn9I9U/Q6tegRYpii4ys\r\n" -"c176NmU+umy6XmuSKV5qD9bSpZWG2nLFnslrN15Lm3fhZxoeMNhBaEDTnLT26yoi\r\n" -"33gp0mSSWy94ZEqipms+ULF6sY1ZtFW6tpGFoy8CgYAQHhnnvJflIs2ky4q10B60\r\n" -"ZcxFp3rtDpkp0JxhFLhiizFrujMtZSjYNm5U7KkgPVHhLELEUvCmOnKTt4ap/vZ0\r\n" -"BxJNe1GZH3pW6SAvGDQpl9sG7uu/vTFP+lCxukmzxB0DrrDcvorEkKMom7ZCCRvW\r\n" -"KZsZ6YeH2Z81BauRj218kQKBgQCUV/DgKP2985xDTT79N08jUo3hTP5MVYCCuj/+\r\n" -"UeEw1TvZcx3LJby7P6Xad6a1/BqveaGyFKIfEFIaBUBItk801sDDpDaYc4gL00Xc\r\n" -"7lFuBHOZkxJYlss5QrGpuOEl9ZwUt5IrFLBdYaKqNHzNVC1pCPfb/JyH6Dr2HUxq\r\n" -"gxUwAQKBgQCcU6G2L8AG9d9c0UpOyL1tMvFe5Ttw0KjlQVdsh1MP6yigYo9DYuwu\r\n" -"bHFVW2r0dBTqegP2/KTOxKzaHfC1qf0RGDsUoJCNJrd1cwoCLG8P2EF4w3OBrKqv\r\n" -"8u4ytY0F+Vlanj5lm3TaoHSVF1+NWPyOTiwevIECGKwSxvlki4fDAA==\r\n" -"-----END RSA PRIVATE KEY-----\r\n"; -const size_t mbedtls_test_cli_key_rsa_len = sizeof( mbedtls_test_cli_key_rsa ); #endif /* MBEDTLS_RSA_C */ -#if defined(MBEDTLS_PEM_PARSE_C) -/* Concatenation of all available CA certificates */ -const char mbedtls_test_cas_pem[] = -#ifdef TEST_CA_CRT_RSA_SHA1 - TEST_CA_CRT_RSA_SHA1 -#endif -#ifdef TEST_CA_CRT_RSA_SHA256 - TEST_CA_CRT_RSA_SHA256 -#endif -#ifdef TEST_CA_CRT_EC - TEST_CA_CRT_EC -#endif - ""; -const size_t mbedtls_test_cas_pem_len = sizeof( mbedtls_test_cas_pem ); -#endif +/* API stability forces us to declare + * mbedtls_test_{ca|srv|cli}_{key|pwd|crt} + * as pointers. */ +static const char test_ca_key[] = TEST_CA_KEY; +static const char test_ca_pwd[] = TEST_CA_PWD; +static const char test_ca_crt[] = TEST_CA_CRT; -/* List of all available CA certificates */ +static const char test_srv_key[] = TEST_SRV_KEY; +static const char test_srv_pwd[] = TEST_SRV_PWD; +static const char test_srv_crt[] = TEST_SRV_CRT; + +static const char test_cli_key[] = TEST_CLI_KEY; +static const char test_cli_pwd[] = TEST_CLI_PWD; +static const char test_cli_crt[] = TEST_CLI_CRT; + +const char *mbedtls_test_ca_key = test_ca_key; +const char *mbedtls_test_ca_pwd = test_ca_pwd; +const char *mbedtls_test_ca_crt = test_ca_crt; + +const char *mbedtls_test_srv_key = test_srv_key; +const char *mbedtls_test_srv_pwd = test_srv_pwd; +const char *mbedtls_test_srv_crt = test_srv_crt; + +const char *mbedtls_test_cli_key = test_cli_key; +const char *mbedtls_test_cli_pwd = test_cli_pwd; +const char *mbedtls_test_cli_crt = test_cli_crt; + +const size_t mbedtls_test_ca_key_len = + sizeof( test_ca_key ); +const size_t mbedtls_test_ca_pwd_len = + sizeof( test_ca_pwd ) - 1; +const size_t mbedtls_test_ca_crt_len = + sizeof( test_ca_crt ); + +const size_t mbedtls_test_srv_key_len = + sizeof( test_srv_key ); +const size_t mbedtls_test_srv_pwd_len = + sizeof( test_srv_pwd ) - 1; +const size_t mbedtls_test_srv_crt_len = + sizeof( test_srv_crt ); + +const size_t mbedtls_test_cli_key_len = + sizeof( test_cli_key ); +const size_t mbedtls_test_cli_pwd_len = + sizeof( test_cli_pwd ) - 1; +const size_t mbedtls_test_cli_crt_len = + sizeof( test_cli_crt ); + +/* + * + * Lists of certificates + * + */ + +/* List of CAs in PEM or DER, depending on config */ const char * mbedtls_test_cas[] = { -#if defined(TEST_CA_CRT_RSA_SHA1) +#if defined(MBEDTLS_RSA_C) && defined(MBEDTLS_SHA1_C) mbedtls_test_ca_crt_rsa_sha1, #endif -#if defined(TEST_CA_CRT_RSA_SHA256) +#if defined(MBEDTLS_RSA_C) && defined(MBEDTLS_SHA256_C) mbedtls_test_ca_crt_rsa_sha256, #endif #if defined(MBEDTLS_ECDSA_C) @@ -359,10 +1704,10 @@ const char * mbedtls_test_cas[] = { NULL }; const size_t mbedtls_test_cas_len[] = { -#if defined(TEST_CA_CRT_RSA_SHA1) +#if defined(MBEDTLS_RSA_C) && defined(MBEDTLS_SHA1_C) sizeof( mbedtls_test_ca_crt_rsa_sha1 ), #endif -#if defined(TEST_CA_CRT_RSA_SHA256) +#if defined(MBEDTLS_RSA_C) && defined(MBEDTLS_SHA256_C) sizeof( mbedtls_test_ca_crt_rsa_sha256 ), #endif #if defined(MBEDTLS_ECDSA_C) @@ -371,36 +1716,53 @@ const size_t mbedtls_test_cas_len[] = { 0 }; +/* List of all available CA certificates in DER format */ +const unsigned char * mbedtls_test_cas_der[] = { #if defined(MBEDTLS_RSA_C) -const char *mbedtls_test_ca_crt = mbedtls_test_ca_crt_rsa; /* SHA1 or SHA256 */ -const char *mbedtls_test_ca_key = mbedtls_test_ca_key_rsa; -const char *mbedtls_test_ca_pwd = mbedtls_test_ca_pwd_rsa; -const char *mbedtls_test_srv_crt = mbedtls_test_srv_crt_rsa; -const char *mbedtls_test_srv_key = mbedtls_test_srv_key_rsa; -const char *mbedtls_test_cli_crt = mbedtls_test_cli_crt_rsa; -const char *mbedtls_test_cli_key = mbedtls_test_cli_key_rsa; -const size_t mbedtls_test_ca_crt_len = sizeof( mbedtls_test_ca_crt_rsa ); -const size_t mbedtls_test_ca_key_len = sizeof( mbedtls_test_ca_key_rsa ); -const size_t mbedtls_test_ca_pwd_len = sizeof( mbedtls_test_ca_pwd_rsa ) - 1; -const size_t mbedtls_test_srv_crt_len = sizeof( mbedtls_test_srv_crt_rsa ); -const size_t mbedtls_test_srv_key_len = sizeof( mbedtls_test_srv_key_rsa ); -const size_t mbedtls_test_cli_crt_len = sizeof( mbedtls_test_cli_crt_rsa ); -const size_t mbedtls_test_cli_key_len = sizeof( mbedtls_test_cli_key_rsa ); -#else /* ! MBEDTLS_RSA_C, so MBEDTLS_ECDSA_C */ -const char *mbedtls_test_ca_crt = mbedtls_test_ca_crt_ec; -const char *mbedtls_test_ca_key = mbedtls_test_ca_key_ec; -const char *mbedtls_test_ca_pwd = mbedtls_test_ca_pwd_ec; -const char *mbedtls_test_srv_crt = mbedtls_test_srv_crt_ec; -const char *mbedtls_test_srv_key = mbedtls_test_srv_key_ec; -const char *mbedtls_test_cli_crt = mbedtls_test_cli_crt_ec; -const char *mbedtls_test_cli_key = mbedtls_test_cli_key_ec; -const size_t mbedtls_test_ca_crt_len = sizeof( mbedtls_test_ca_crt_ec ); -const size_t mbedtls_test_ca_key_len = sizeof( mbedtls_test_ca_key_ec ); -const size_t mbedtls_test_ca_pwd_len = sizeof( mbedtls_test_ca_pwd_ec ) - 1; -const size_t mbedtls_test_srv_crt_len = sizeof( mbedtls_test_srv_crt_ec ); -const size_t mbedtls_test_srv_key_len = sizeof( mbedtls_test_srv_key_ec ); -const size_t mbedtls_test_cli_crt_len = sizeof( mbedtls_test_cli_crt_ec ); -const size_t mbedtls_test_cli_key_len = sizeof( mbedtls_test_cli_key_ec ); +#if defined(MBEDTLS_SHA256_C) + mbedtls_test_ca_crt_rsa_sha256_der, +#endif /* MBEDTLS_SHA256_C */ +#if defined(MBEDTLS_SHA1_C) + mbedtls_test_ca_crt_rsa_sha1_der, +#endif /* MBEDTLS_SHA1_C */ #endif /* MBEDTLS_RSA_C */ +#if defined(MBEDTLS_ECDSA_C) + mbedtls_test_ca_crt_ec_der, +#endif /* MBEDTLS_ECDSA_C */ + NULL +}; + +const size_t mbedtls_test_cas_der_len[] = { +#if defined(MBEDTLS_RSA_C) +#if defined(MBEDTLS_SHA256_C) + sizeof( mbedtls_test_ca_crt_rsa_sha256_der ), +#endif /* MBEDTLS_SHA256_C */ +#if defined(MBEDTLS_SHA1_C) + sizeof( mbedtls_test_ca_crt_rsa_sha1_der ), +#endif /* MBEDTLS_SHA1_C */ +#endif /* MBEDTLS_RSA_C */ +#if defined(MBEDTLS_ECDSA_C) + sizeof( mbedtls_test_ca_crt_ec_der ), +#endif /* MBEDTLS_ECDSA_C */ + 0 +}; + +/* Concatenation of all available CA certificates in PEM format */ +#if defined(MBEDTLS_PEM_PARSE_C) +const char mbedtls_test_cas_pem[] = +#if defined(MBEDTLS_RSA_C) +#if defined(MBEDTLS_SHA256_C) + TEST_CA_CRT_RSA_SHA256_PEM +#endif /* MBEDTLS_SHA256_C */ +#if defined(MBEDTLS_SHA1_C) + TEST_CA_CRT_RSA_SHA1_PEM +#endif /* MBEDTLS_SHA1_C */ +#endif /* MBEDTLS_RSA_C */ +#if defined(MBEDTLS_ECDSA_C) + TEST_CA_CRT_EC_PEM +#endif /* MBEDTLS_ECDSA_C */ + ""; +const size_t mbedtls_test_cas_pem_len = sizeof( mbedtls_test_cas_pem ); +#endif /* MBEDTLS_PEM_PARSE_C */ #endif /* MBEDTLS_CERTS_C */ diff --git a/thirdparty/mbedtls/library/chacha20.c b/thirdparty/mbedtls/library/chacha20.c index 0757163e2f7..8a3610f0e03 100644 --- a/thirdparty/mbedtls/library/chacha20.c +++ b/thirdparty/mbedtls/library/chacha20.c @@ -60,14 +60,14 @@ MBEDTLS_INTERNAL_VALIDATE( cond ) #define BYTES_TO_U32_LE( data, offset ) \ - ( (uint32_t) data[offset] \ - | (uint32_t) ( (uint32_t) data[( offset ) + 1] << 8 ) \ - | (uint32_t) ( (uint32_t) data[( offset ) + 2] << 16 ) \ - | (uint32_t) ( (uint32_t) data[( offset ) + 3] << 24 ) \ + ( (uint32_t) (data)[offset] \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 1] << 8 ) \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 2] << 16 ) \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 3] << 24 ) \ ) #define ROTL32( value, amount ) \ - ( (uint32_t) ( value << amount ) | ( value >> ( 32 - amount ) ) ) + ( (uint32_t) ( (value) << (amount) ) | ( (value) >> ( 32 - (amount) ) ) ) #define CHACHA20_CTR_INDEX ( 12U ) diff --git a/thirdparty/mbedtls/library/debug.c b/thirdparty/mbedtls/library/debug.c index 824cd0236ed..36510cdd56d 100644 --- a/thirdparty/mbedtls/library/debug.c +++ b/thirdparty/mbedtls/library/debug.c @@ -86,8 +86,13 @@ void mbedtls_debug_print_msg( const mbedtls_ssl_context *ssl, int level, char str[DEBUG_BUF_SIZE]; int ret; - if( NULL == ssl || NULL == ssl->conf || NULL == ssl->conf->f_dbg || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + level > debug_threshold ) + { return; + } va_start( argp, format ); #if defined(_WIN32) @@ -121,8 +126,13 @@ void mbedtls_debug_print_ret( const mbedtls_ssl_context *ssl, int level, { char str[DEBUG_BUF_SIZE]; - if( ssl->conf == NULL || ssl->conf->f_dbg == NULL || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + level > debug_threshold ) + { return; + } /* * With non-blocking I/O and examples that just retry immediately, @@ -146,8 +156,13 @@ void mbedtls_debug_print_buf( const mbedtls_ssl_context *ssl, int level, char txt[17]; size_t i, idx = 0; - if( ssl->conf == NULL || ssl->conf->f_dbg == NULL || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + level > debug_threshold ) + { return; + } mbedtls_snprintf( str + idx, sizeof( str ) - idx, "dumping '%s' (%u bytes)\n", text, (unsigned int) len ); @@ -199,8 +214,13 @@ void mbedtls_debug_print_ecp( const mbedtls_ssl_context *ssl, int level, { char str[DEBUG_BUF_SIZE]; - if( ssl->conf == NULL || ssl->conf->f_dbg == NULL || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + level > debug_threshold ) + { return; + } mbedtls_snprintf( str, sizeof( str ), "%s(X)", text ); mbedtls_debug_print_mpi( ssl, level, file, line, str, &X->X ); @@ -219,8 +239,14 @@ void mbedtls_debug_print_mpi( const mbedtls_ssl_context *ssl, int level, int j, k, zeros = 1; size_t i, n, idx = 0; - if( ssl->conf == NULL || ssl->conf->f_dbg == NULL || X == NULL || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + NULL == X || + level > debug_threshold ) + { return; + } for( n = X->n - 1; n > 0; n-- ) if( X->p[n] != 0 ) @@ -345,8 +371,14 @@ void mbedtls_debug_print_crt( const mbedtls_ssl_context *ssl, int level, char str[DEBUG_BUF_SIZE]; int i = 0; - if( ssl->conf == NULL || ssl->conf->f_dbg == NULL || crt == NULL || level > debug_threshold ) + if( NULL == ssl || + NULL == ssl->conf || + NULL == ssl->conf->f_dbg || + NULL == crt || + level > debug_threshold ) + { return; + } while( crt != NULL ) { diff --git a/thirdparty/mbedtls/library/des.c b/thirdparty/mbedtls/library/des.c index ca9e071f325..8a33d82e50e 100644 --- a/thirdparty/mbedtls/library/des.c +++ b/thirdparty/mbedtls/library/des.c @@ -257,50 +257,57 @@ static const uint32_t RHs[16] = /* * Initial Permutation macro */ -#define DES_IP(X,Y) \ -{ \ - T = ((X >> 4) ^ Y) & 0x0F0F0F0F; Y ^= T; X ^= (T << 4); \ - T = ((X >> 16) ^ Y) & 0x0000FFFF; Y ^= T; X ^= (T << 16); \ - T = ((Y >> 2) ^ X) & 0x33333333; X ^= T; Y ^= (T << 2); \ - T = ((Y >> 8) ^ X) & 0x00FF00FF; X ^= T; Y ^= (T << 8); \ - Y = ((Y << 1) | (Y >> 31)) & 0xFFFFFFFF; \ - T = (X ^ Y) & 0xAAAAAAAA; Y ^= T; X ^= T; \ - X = ((X << 1) | (X >> 31)) & 0xFFFFFFFF; \ -} +#define DES_IP(X,Y) \ + do \ + { \ + T = (((X) >> 4) ^ (Y)) & 0x0F0F0F0F; (Y) ^= T; (X) ^= (T << 4); \ + T = (((X) >> 16) ^ (Y)) & 0x0000FFFF; (Y) ^= T; (X) ^= (T << 16); \ + T = (((Y) >> 2) ^ (X)) & 0x33333333; (X) ^= T; (Y) ^= (T << 2); \ + T = (((Y) >> 8) ^ (X)) & 0x00FF00FF; (X) ^= T; (Y) ^= (T << 8); \ + (Y) = (((Y) << 1) | ((Y) >> 31)) & 0xFFFFFFFF; \ + T = ((X) ^ (Y)) & 0xAAAAAAAA; (Y) ^= T; (X) ^= T; \ + (X) = (((X) << 1) | ((X) >> 31)) & 0xFFFFFFFF; \ + } while( 0 ) /* * Final Permutation macro */ -#define DES_FP(X,Y) \ -{ \ - X = ((X << 31) | (X >> 1)) & 0xFFFFFFFF; \ - T = (X ^ Y) & 0xAAAAAAAA; X ^= T; Y ^= T; \ - Y = ((Y << 31) | (Y >> 1)) & 0xFFFFFFFF; \ - T = ((Y >> 8) ^ X) & 0x00FF00FF; X ^= T; Y ^= (T << 8); \ - T = ((Y >> 2) ^ X) & 0x33333333; X ^= T; Y ^= (T << 2); \ - T = ((X >> 16) ^ Y) & 0x0000FFFF; Y ^= T; X ^= (T << 16); \ - T = ((X >> 4) ^ Y) & 0x0F0F0F0F; Y ^= T; X ^= (T << 4); \ -} +#define DES_FP(X,Y) \ + do \ + { \ + (X) = (((X) << 31) | ((X) >> 1)) & 0xFFFFFFFF; \ + T = ((X) ^ (Y)) & 0xAAAAAAAA; (X) ^= T; (Y) ^= T; \ + (Y) = (((Y) << 31) | ((Y) >> 1)) & 0xFFFFFFFF; \ + T = (((Y) >> 8) ^ (X)) & 0x00FF00FF; (X) ^= T; (Y) ^= (T << 8); \ + T = (((Y) >> 2) ^ (X)) & 0x33333333; (X) ^= T; (Y) ^= (T << 2); \ + T = (((X) >> 16) ^ (Y)) & 0x0000FFFF; (Y) ^= T; (X) ^= (T << 16); \ + T = (((X) >> 4) ^ (Y)) & 0x0F0F0F0F; (Y) ^= T; (X) ^= (T << 4); \ + } while( 0 ) /* * DES round macro */ -#define DES_ROUND(X,Y) \ -{ \ - T = *SK++ ^ X; \ - Y ^= SB8[ (T ) & 0x3F ] ^ \ - SB6[ (T >> 8) & 0x3F ] ^ \ - SB4[ (T >> 16) & 0x3F ] ^ \ - SB2[ (T >> 24) & 0x3F ]; \ - \ - T = *SK++ ^ ((X << 28) | (X >> 4)); \ - Y ^= SB7[ (T ) & 0x3F ] ^ \ - SB5[ (T >> 8) & 0x3F ] ^ \ - SB3[ (T >> 16) & 0x3F ] ^ \ - SB1[ (T >> 24) & 0x3F ]; \ -} +#define DES_ROUND(X,Y) \ + do \ + { \ + T = *SK++ ^ (X); \ + (Y) ^= SB8[ (T ) & 0x3F ] ^ \ + SB6[ (T >> 8) & 0x3F ] ^ \ + SB4[ (T >> 16) & 0x3F ] ^ \ + SB2[ (T >> 24) & 0x3F ]; \ + \ + T = *SK++ ^ (((X) << 28) | ((X) >> 4)); \ + (Y) ^= SB7[ (T ) & 0x3F ] ^ \ + SB5[ (T >> 8) & 0x3F ] ^ \ + SB3[ (T >> 16) & 0x3F ] ^ \ + SB1[ (T >> 24) & 0x3F ]; \ + } while( 0 ) -#define SWAP(a,b) { uint32_t t = a; a = b; b = t; t = 0; } +#define SWAP(a,b) \ + do \ + { \ + uint32_t t = (a); (a) = (b); (b) = t; t = 0; \ + } while( 0 ) void mbedtls_des_init( mbedtls_des_context *ctx ) { diff --git a/thirdparty/mbedtls/library/dhm.c b/thirdparty/mbedtls/library/dhm.c index fb6937e8549..8255632a993 100644 --- a/thirdparty/mbedtls/library/dhm.c +++ b/thirdparty/mbedtls/library/dhm.c @@ -649,12 +649,28 @@ int mbedtls_dhm_parse_dhmfile( mbedtls_dhm_context *dhm, const char *path ) #if defined(MBEDTLS_SELF_TEST) +#if defined(MBEDTLS_PEM_PARSE_C) static const char mbedtls_test_dhm_params[] = "-----BEGIN DH PARAMETERS-----\r\n" "MIGHAoGBAJ419DBEOgmQTzo5qXl5fQcN9TN455wkOL7052HzxxRVMyhYmwQcgJvh\r\n" "1sa18fyfR9OiVEMYglOpkqVoGLN7qd5aQNNi5W7/C+VBdHTBJcGZJyyP5B3qcz32\r\n" "9mLJKudlVudV0Qxk5qUJaPZ/xupz0NyoVpviuiBOI1gNi8ovSXWzAgEC\r\n" "-----END DH PARAMETERS-----\r\n"; +#else /* MBEDTLS_PEM_PARSE_C */ +static const char mbedtls_test_dhm_params[] = { + 0x30, 0x81, 0x87, 0x02, 0x81, 0x81, 0x00, 0x9e, 0x35, 0xf4, 0x30, 0x44, + 0x3a, 0x09, 0x90, 0x4f, 0x3a, 0x39, 0xa9, 0x79, 0x79, 0x7d, 0x07, 0x0d, + 0xf5, 0x33, 0x78, 0xe7, 0x9c, 0x24, 0x38, 0xbe, 0xf4, 0xe7, 0x61, 0xf3, + 0xc7, 0x14, 0x55, 0x33, 0x28, 0x58, 0x9b, 0x04, 0x1c, 0x80, 0x9b, 0xe1, + 0xd6, 0xc6, 0xb5, 0xf1, 0xfc, 0x9f, 0x47, 0xd3, 0xa2, 0x54, 0x43, 0x18, + 0x82, 0x53, 0xa9, 0x92, 0xa5, 0x68, 0x18, 0xb3, 0x7b, 0xa9, 0xde, 0x5a, + 0x40, 0xd3, 0x62, 0xe5, 0x6e, 0xff, 0x0b, 0xe5, 0x41, 0x74, 0x74, 0xc1, + 0x25, 0xc1, 0x99, 0x27, 0x2c, 0x8f, 0xe4, 0x1d, 0xea, 0x73, 0x3d, 0xf6, + 0xf6, 0x62, 0xc9, 0x2a, 0xe7, 0x65, 0x56, 0xe7, 0x55, 0xd1, 0x0c, 0x64, + 0xe6, 0xa5, 0x09, 0x68, 0xf6, 0x7f, 0xc6, 0xea, 0x73, 0xd0, 0xdc, 0xa8, + 0x56, 0x9b, 0xe2, 0xba, 0x20, 0x4e, 0x23, 0x58, 0x0d, 0x8b, 0xca, 0x2f, + 0x49, 0x75, 0xb3, 0x02, 0x01, 0x02 }; +#endif /* MBEDTLS_PEM_PARSE_C */ static const size_t mbedtls_test_dhm_params_len = sizeof( mbedtls_test_dhm_params ); diff --git a/thirdparty/mbedtls/library/ecdh.c b/thirdparty/mbedtls/library/ecdh.c index da95c60dad2..c5726877d5e 100644 --- a/thirdparty/mbedtls/library/ecdh.c +++ b/thirdparty/mbedtls/library/ecdh.c @@ -49,6 +49,16 @@ typedef mbedtls_ecdh_context mbedtls_ecdh_context_mbed; #endif +static mbedtls_ecp_group_id mbedtls_ecdh_grp_id( + const mbedtls_ecdh_context *ctx ) +{ +#if defined(MBEDTLS_ECDH_LEGACY_CONTEXT) + return( ctx->grp.id ); +#else + return( ctx->grp_id ); +#endif +} + #if !defined(MBEDTLS_ECDH_GEN_PUBLIC_ALT) /* * Generate public key (restartable version) @@ -442,8 +452,21 @@ int mbedtls_ecdh_get_params( mbedtls_ecdh_context *ctx, ECDH_VALIDATE_RET( side == MBEDTLS_ECDH_OURS || side == MBEDTLS_ECDH_THEIRS ); - if( ( ret = mbedtls_ecdh_setup( ctx, key->grp.id ) ) != 0 ) - return( ret ); + if( mbedtls_ecdh_grp_id( ctx ) == MBEDTLS_ECP_DP_NONE ) + { + /* This is the first call to get_params(). Set up the context + * for use with the group. */ + if( ( ret = mbedtls_ecdh_setup( ctx, key->grp.id ) ) != 0 ) + return( ret ); + } + else + { + /* This is not the first call to get_params(). Check that the + * current key's group is the same as the context's, which was set + * from the first key's group. */ + if( mbedtls_ecdh_grp_id( ctx ) != key->grp.id ) + return( MBEDTLS_ERR_ECP_BAD_INPUT_DATA ); + } #if defined(MBEDTLS_ECDH_LEGACY_CONTEXT) return( ecdh_get_params_internal( ctx, key, side ) ); diff --git a/thirdparty/mbedtls/library/ecdsa.c b/thirdparty/mbedtls/library/ecdsa.c index 1204ef9949a..dc19384d614 100644 --- a/thirdparty/mbedtls/library/ecdsa.c +++ b/thirdparty/mbedtls/library/ecdsa.c @@ -800,11 +800,16 @@ cleanup: int mbedtls_ecdsa_genkey( mbedtls_ecdsa_context *ctx, mbedtls_ecp_group_id gid, int (*f_rng)(void *, unsigned char *, size_t), void *p_rng ) { + int ret = 0; ECDSA_VALIDATE_RET( ctx != NULL ); ECDSA_VALIDATE_RET( f_rng != NULL ); - return( mbedtls_ecp_group_load( &ctx->grp, gid ) || - mbedtls_ecp_gen_keypair( &ctx->grp, &ctx->d, &ctx->Q, f_rng, p_rng ) ); + ret = mbedtls_ecp_group_load( &ctx->grp, gid ); + if( ret != 0 ) + return( ret ); + + return( mbedtls_ecp_gen_keypair( &ctx->grp, &ctx->d, + &ctx->Q, f_rng, p_rng ) ); } #endif /* !MBEDTLS_ECDSA_GENKEY_ALT */ diff --git a/thirdparty/mbedtls/library/ecp.c b/thirdparty/mbedtls/library/ecp.c index ecea5910e07..db36191b9b8 100644 --- a/thirdparty/mbedtls/library/ecp.c +++ b/thirdparty/mbedtls/library/ecp.c @@ -1046,25 +1046,29 @@ cleanup: #define INC_MUL_COUNT #endif -#define MOD_MUL( N ) do { MBEDTLS_MPI_CHK( ecp_modp( &N, grp ) ); INC_MUL_COUNT } \ - while( 0 ) +#define MOD_MUL( N ) \ + do \ + { \ + MBEDTLS_MPI_CHK( ecp_modp( &(N), grp ) ); \ + INC_MUL_COUNT \ + } while( 0 ) /* * Reduce a mbedtls_mpi mod p in-place, to use after mbedtls_mpi_sub_mpi * N->s < 0 is a very fast test, which fails only if N is 0 */ -#define MOD_SUB( N ) \ - while( N.s < 0 && mbedtls_mpi_cmp_int( &N, 0 ) != 0 ) \ - MBEDTLS_MPI_CHK( mbedtls_mpi_add_mpi( &N, &N, &grp->P ) ) +#define MOD_SUB( N ) \ + while( (N).s < 0 && mbedtls_mpi_cmp_int( &(N), 0 ) != 0 ) \ + MBEDTLS_MPI_CHK( mbedtls_mpi_add_mpi( &(N), &(N), &grp->P ) ) /* * Reduce a mbedtls_mpi mod p in-place, to use after mbedtls_mpi_add_mpi and mbedtls_mpi_mul_int. * We known P, N and the result are positive, so sub_abs is correct, and * a bit faster. */ -#define MOD_ADD( N ) \ - while( mbedtls_mpi_cmp_mpi( &N, &grp->P ) >= 0 ) \ - MBEDTLS_MPI_CHK( mbedtls_mpi_sub_abs( &N, &N, &grp->P ) ) +#define MOD_ADD( N ) \ + while( mbedtls_mpi_cmp_mpi( &(N), &grp->P ) >= 0 ) \ + MBEDTLS_MPI_CHK( mbedtls_mpi_sub_abs( &(N), &(N), &grp->P ) ) #if defined(ECP_SHORTWEIERSTRASS) /* diff --git a/thirdparty/mbedtls/library/ecp_curves.c b/thirdparty/mbedtls/library/ecp_curves.c index 731621dc3ca..282481d0538 100644 --- a/thirdparty/mbedtls/library/ecp_curves.c +++ b/thirdparty/mbedtls/library/ecp_curves.c @@ -51,11 +51,11 @@ */ #if defined(MBEDTLS_HAVE_INT32) -#define BYTES_TO_T_UINT_4( a, b, c, d ) \ - ( (mbedtls_mpi_uint) a << 0 ) | \ - ( (mbedtls_mpi_uint) b << 8 ) | \ - ( (mbedtls_mpi_uint) c << 16 ) | \ - ( (mbedtls_mpi_uint) d << 24 ) +#define BYTES_TO_T_UINT_4( a, b, c, d ) \ + ( (mbedtls_mpi_uint) (a) << 0 ) | \ + ( (mbedtls_mpi_uint) (b) << 8 ) | \ + ( (mbedtls_mpi_uint) (c) << 16 ) | \ + ( (mbedtls_mpi_uint) (d) << 24 ) #define BYTES_TO_T_UINT_2( a, b ) \ BYTES_TO_T_UINT_4( a, b, 0, 0 ) @@ -67,14 +67,14 @@ #else /* 64-bits */ #define BYTES_TO_T_UINT_8( a, b, c, d, e, f, g, h ) \ - ( (mbedtls_mpi_uint) a << 0 ) | \ - ( (mbedtls_mpi_uint) b << 8 ) | \ - ( (mbedtls_mpi_uint) c << 16 ) | \ - ( (mbedtls_mpi_uint) d << 24 ) | \ - ( (mbedtls_mpi_uint) e << 32 ) | \ - ( (mbedtls_mpi_uint) f << 40 ) | \ - ( (mbedtls_mpi_uint) g << 48 ) | \ - ( (mbedtls_mpi_uint) h << 56 ) + ( (mbedtls_mpi_uint) (a) << 0 ) | \ + ( (mbedtls_mpi_uint) (b) << 8 ) | \ + ( (mbedtls_mpi_uint) (c) << 16 ) | \ + ( (mbedtls_mpi_uint) (d) << 24 ) | \ + ( (mbedtls_mpi_uint) (e) << 32 ) | \ + ( (mbedtls_mpi_uint) (f) << 40 ) | \ + ( (mbedtls_mpi_uint) (g) << 48 ) | \ + ( (mbedtls_mpi_uint) (h) << 56 ) #define BYTES_TO_T_UINT_4( a, b, c, d ) \ BYTES_TO_T_UINT_8( a, b, c, d, 0, 0, 0, 0 ) @@ -890,7 +890,7 @@ static inline void carry64( mbedtls_mpi_uint *dst, mbedtls_mpi_uint *carry ) } #define WIDTH 8 / sizeof( mbedtls_mpi_uint ) -#define A( i ) N->p + i * WIDTH +#define A( i ) N->p + (i) * WIDTH #define ADD( i ) add64( p, A( i ), &c ) #define NEXT p += WIDTH; carry64( p, &c ) #define LAST p += WIDTH; *p = c; while( ++p < end ) *p = 0 @@ -955,7 +955,8 @@ cleanup: #else /* 64-bit */ #define MAX32 N->n * 2 -#define A( j ) j % 2 ? (uint32_t)( N->p[j/2] >> 32 ) : (uint32_t)( N->p[j/2] ) +#define A( j ) (j) % 2 ? (uint32_t)( N->p[(j)/2] >> 32 ) : \ + (uint32_t)( N->p[(j)/2] ) #define STORE32 \ if( i % 2 ) { \ N->p[i/2] &= 0x00000000FFFFFFFF; \ @@ -989,20 +990,21 @@ static inline void sub32( uint32_t *dst, uint32_t src, signed char *carry ) * Helpers for the main 'loop' * (see fix_negative for the motivation of C) */ -#define INIT( b ) \ - int ret; \ - signed char c = 0, cc; \ - uint32_t cur; \ - size_t i = 0, bits = b; \ - mbedtls_mpi C; \ - mbedtls_mpi_uint Cp[ b / 8 / sizeof( mbedtls_mpi_uint) + 1 ]; \ - \ - C.s = 1; \ - C.n = b / 8 / sizeof( mbedtls_mpi_uint) + 1; \ - C.p = Cp; \ - memset( Cp, 0, C.n * sizeof( mbedtls_mpi_uint ) ); \ - \ - MBEDTLS_MPI_CHK( mbedtls_mpi_grow( N, b * 2 / 8 / sizeof( mbedtls_mpi_uint ) ) ); \ +#define INIT( b ) \ + int ret; \ + signed char c = 0, cc; \ + uint32_t cur; \ + size_t i = 0, bits = (b); \ + mbedtls_mpi C; \ + mbedtls_mpi_uint Cp[ (b) / 8 / sizeof( mbedtls_mpi_uint) + 1 ]; \ + \ + C.s = 1; \ + C.n = (b) / 8 / sizeof( mbedtls_mpi_uint) + 1; \ + C.p = Cp; \ + memset( Cp, 0, C.n * sizeof( mbedtls_mpi_uint ) ); \ + \ + MBEDTLS_MPI_CHK( mbedtls_mpi_grow( N, (b) * 2 / 8 / \ + sizeof( mbedtls_mpi_uint ) ) ); \ LOAD32; #define NEXT \ diff --git a/thirdparty/mbedtls/library/entropy_poll.c b/thirdparty/mbedtls/library/entropy_poll.c index ba56b70f77e..4556f88a557 100644 --- a/thirdparty/mbedtls/library/entropy_poll.c +++ b/thirdparty/mbedtls/library/entropy_poll.c @@ -61,43 +61,28 @@ #define _WIN32_WINNT 0x0400 #endif #include -#include -#if defined(_MSC_VER) && _MSC_VER <= 1600 -/* Visual Studio 2010 and earlier issue a warning when both and - * are included, as they redefine a number of _MAX constants. - * These constants are guaranteed to be the same, though, so we suppress the - * warning when including intsafe.h. - */ -#pragma warning( push ) -#pragma warning( disable : 4005 ) -#endif -#include -#if defined(_MSC_VER) && _MSC_VER <= 1600 -#pragma warning( pop ) -#endif +#include int mbedtls_platform_entropy_poll( void *data, unsigned char *output, size_t len, size_t *olen ) { - ULONG len_as_ulong = 0; + HCRYPTPROV provider; ((void) data); *olen = 0; - /* - * BCryptGenRandom takes ULONG for size, which is smaller than size_t on - * 64-bit Windows platforms. Ensure len's value can be safely converted into - * a ULONG. - */ - if ( FAILED( SizeTToULong( len, &len_as_ulong ) ) ) + if( CryptAcquireContext( &provider, NULL, NULL, + PROV_RSA_FULL, CRYPT_VERIFYCONTEXT ) == FALSE ) { return( MBEDTLS_ERR_ENTROPY_SOURCE_FAILED ); } - if ( !BCRYPT_SUCCESS( BCryptGenRandom( NULL, output, len_as_ulong, BCRYPT_USE_SYSTEM_PREFERRED_RNG ) ) ) + if( CryptGenRandom( provider, (DWORD) len, output ) == FALSE ) { + CryptReleaseContext( provider, 0 ); return( MBEDTLS_ERR_ENTROPY_SOURCE_FAILED ); } + CryptReleaseContext( provider, 0 ); *olen = len; return( 0 ); diff --git a/thirdparty/mbedtls/library/havege.c b/thirdparty/mbedtls/library/havege.c index 4dcac028755..54f897c6e7a 100644 --- a/thirdparty/mbedtls/library/havege.c +++ b/thirdparty/mbedtls/library/havege.c @@ -54,7 +54,7 @@ * ------------------------------------------------------------------------ */ -#define SWAP(X,Y) { int *T = X; X = Y; Y = T; } +#define SWAP(X,Y) { int *T = (X); (X) = (Y); (Y) = T; } #define TST1_ENTER if( PTEST & 1 ) { PTEST ^= 3; PTEST >>= 1; #define TST2_ENTER if( PTEST & 1 ) { PTEST ^= 3; PTEST >>= 1; diff --git a/thirdparty/mbedtls/library/md4.c b/thirdparty/mbedtls/library/md4.c index 3f8ddff31d4..828fd42999a 100644 --- a/thirdparty/mbedtls/library/md4.c +++ b/thirdparty/mbedtls/library/md4.c @@ -137,15 +137,21 @@ int mbedtls_internal_md4_process( mbedtls_md4_context *ctx, GET_UINT32_LE( X[14], data, 56 ); GET_UINT32_LE( X[15], data, 60 ); -#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) +#define S(x,n) (((x) << (n)) | (((x) & 0xFFFFFFFF) >> (32 - (n)))) A = ctx->state[0]; B = ctx->state[1]; C = ctx->state[2]; D = ctx->state[3]; -#define F(x, y, z) ((x & y) | ((~x) & z)) -#define P(a,b,c,d,x,s) { a += F(b,c,d) + x; a = S(a,s); } +#define F(x, y, z) (((x) & (y)) | ((~(x)) & (z))) +#define P(a,b,c,d,x,s) \ + do \ + { \ + (a) += F((b),(c),(d)) + (x); \ + (a) = S((a),(s)); \ + } while( 0 ) + P( A, B, C, D, X[ 0], 3 ); P( D, A, B, C, X[ 1], 7 ); @@ -167,8 +173,13 @@ int mbedtls_internal_md4_process( mbedtls_md4_context *ctx, #undef P #undef F -#define F(x,y,z) ((x & y) | (x & z) | (y & z)) -#define P(a,b,c,d,x,s) { a += F(b,c,d) + x + 0x5A827999; a = S(a,s); } +#define F(x,y,z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define P(a,b,c,d,x,s) \ + do \ + { \ + (a) += F((b),(c),(d)) + (x) + 0x5A827999; \ + (a) = S((a),(s)); \ + } while( 0 ) P( A, B, C, D, X[ 0], 3 ); P( D, A, B, C, X[ 4], 5 ); @@ -190,8 +201,13 @@ int mbedtls_internal_md4_process( mbedtls_md4_context *ctx, #undef P #undef F -#define F(x,y,z) (x ^ y ^ z) -#define P(a,b,c,d,x,s) { a += F(b,c,d) + x + 0x6ED9EBA1; a = S(a,s); } +#define F(x,y,z) ((x) ^ (y) ^ (z)) +#define P(a,b,c,d,x,s) \ + do \ + { \ + (a) += F((b),(c),(d)) + (x) + 0x6ED9EBA1; \ + (a) = S((a),(s)); \ + } while( 0 ) P( A, B, C, D, X[ 0], 3 ); P( D, A, B, C, X[ 8], 9 ); diff --git a/thirdparty/mbedtls/library/md5.c b/thirdparty/mbedtls/library/md5.c index 2a740cda81e..a93da8a061b 100644 --- a/thirdparty/mbedtls/library/md5.c +++ b/thirdparty/mbedtls/library/md5.c @@ -136,19 +136,22 @@ int mbedtls_internal_md5_process( mbedtls_md5_context *ctx, GET_UINT32_LE( X[14], data, 56 ); GET_UINT32_LE( X[15], data, 60 ); -#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) +#define S(x,n) \ + ( ( (x) << (n) ) | ( ( (x) & 0xFFFFFFFF) >> ( 32 - (n) ) ) ) -#define P(a,b,c,d,k,s,t) \ -{ \ - a += F(b,c,d) + X[k] + t; a = S(a,s) + b; \ -} +#define P(a,b,c,d,k,s,t) \ + do \ + { \ + (a) += F((b),(c),(d)) + X[(k)] + (t); \ + (a) = S((a),(s)) + (b); \ + } while( 0 ) A = ctx->state[0]; B = ctx->state[1]; C = ctx->state[2]; D = ctx->state[3]; -#define F(x,y,z) (z ^ (x & (y ^ z))) +#define F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) P( A, B, C, D, 0, 7, 0xD76AA478 ); P( D, A, B, C, 1, 12, 0xE8C7B756 ); @@ -169,7 +172,7 @@ int mbedtls_internal_md5_process( mbedtls_md5_context *ctx, #undef F -#define F(x,y,z) (y ^ (z & (x ^ y))) +#define F(x,y,z) ((y) ^ ((z) & ((x) ^ (y)))) P( A, B, C, D, 1, 5, 0xF61E2562 ); P( D, A, B, C, 6, 9, 0xC040B340 ); @@ -190,7 +193,7 @@ int mbedtls_internal_md5_process( mbedtls_md5_context *ctx, #undef F -#define F(x,y,z) (x ^ y ^ z) +#define F(x,y,z) ((x) ^ (y) ^ (z)) P( A, B, C, D, 5, 4, 0xFFFA3942 ); P( D, A, B, C, 8, 11, 0x8771F681 ); @@ -211,7 +214,7 @@ int mbedtls_internal_md5_process( mbedtls_md5_context *ctx, #undef F -#define F(x,y,z) (y ^ (x | ~z)) +#define F(x,y,z) ((y) ^ ((x) | ~(z))) P( A, B, C, D, 0, 6, 0xF4292244 ); P( D, A, B, C, 7, 10, 0x432AFF97 ); diff --git a/thirdparty/mbedtls/library/oid.c b/thirdparty/mbedtls/library/oid.c index edea950f8fa..33f437cbe64 100644 --- a/thirdparty/mbedtls/library/oid.c +++ b/thirdparty/mbedtls/library/oid.c @@ -54,22 +54,24 @@ * Macro to generate an internal function for oid_XXX_from_asn1() (used by * the other functions) */ -#define FN_OID_TYPED_FROM_ASN1( TYPE_T, NAME, LIST ) \ -static const TYPE_T * oid_ ## NAME ## _from_asn1( const mbedtls_asn1_buf *oid ) \ -{ \ - const TYPE_T *p = LIST; \ - const mbedtls_oid_descriptor_t *cur = (const mbedtls_oid_descriptor_t *) p; \ - if( p == NULL || oid == NULL ) return( NULL ); \ - while( cur->asn1 != NULL ) { \ - if( cur->asn1_len == oid->len && \ - memcmp( cur->asn1, oid->p, oid->len ) == 0 ) { \ - return( p ); \ - } \ - p++; \ - cur = (const mbedtls_oid_descriptor_t *) p; \ - } \ - return( NULL ); \ -} +#define FN_OID_TYPED_FROM_ASN1( TYPE_T, NAME, LIST ) \ + static const TYPE_T * oid_ ## NAME ## _from_asn1( \ + const mbedtls_asn1_buf *oid ) \ + { \ + const TYPE_T *p = (LIST); \ + const mbedtls_oid_descriptor_t *cur = \ + (const mbedtls_oid_descriptor_t *) p; \ + if( p == NULL || oid == NULL ) return( NULL ); \ + while( cur->asn1 != NULL ) { \ + if( cur->asn1_len == oid->len && \ + memcmp( cur->asn1, oid->p, oid->len ) == 0 ) { \ + return( p ); \ + } \ + p++; \ + cur = (const mbedtls_oid_descriptor_t *) p; \ + } \ + return( NULL ); \ + } /* * Macro to generate a function for retrieving a single attribute from the @@ -103,12 +105,13 @@ int FN_NAME( const mbedtls_asn1_buf *oid, ATTR1_TYPE * ATTR1 ) */ #define FN_OID_GET_ATTR2(FN_NAME, TYPE_T, TYPE_NAME, ATTR1_TYPE, ATTR1, \ ATTR2_TYPE, ATTR2) \ -int FN_NAME( const mbedtls_asn1_buf *oid, ATTR1_TYPE * ATTR1, ATTR2_TYPE * ATTR2 ) \ +int FN_NAME( const mbedtls_asn1_buf *oid, ATTR1_TYPE * ATTR1, \ + ATTR2_TYPE * ATTR2 ) \ { \ const TYPE_T *data = oid_ ## TYPE_NAME ## _from_asn1( oid ); \ - if( data == NULL ) return( MBEDTLS_ERR_OID_NOT_FOUND ); \ - *ATTR1 = data->ATTR1; \ - *ATTR2 = data->ATTR2; \ + if( data == NULL ) return( MBEDTLS_ERR_OID_NOT_FOUND ); \ + *(ATTR1) = data->ATTR1; \ + *(ATTR2) = data->ATTR2; \ return( 0 ); \ } @@ -119,16 +122,16 @@ int FN_NAME( const mbedtls_asn1_buf *oid, ATTR1_TYPE * ATTR1, ATTR2_TYPE * ATTR2 #define FN_OID_GET_OID_BY_ATTR1(FN_NAME, TYPE_T, LIST, ATTR1_TYPE, ATTR1) \ int FN_NAME( ATTR1_TYPE ATTR1, const char **oid, size_t *olen ) \ { \ - const TYPE_T *cur = LIST; \ + const TYPE_T *cur = (LIST); \ while( cur->descriptor.asn1 != NULL ) { \ - if( cur->ATTR1 == ATTR1 ) { \ + if( cur->ATTR1 == (ATTR1) ) { \ *oid = cur->descriptor.asn1; \ *olen = cur->descriptor.asn1_len; \ return( 0 ); \ } \ cur++; \ } \ - return( MBEDTLS_ERR_OID_NOT_FOUND ); \ + return( MBEDTLS_ERR_OID_NOT_FOUND ); \ } /* @@ -140,9 +143,9 @@ int FN_NAME( ATTR1_TYPE ATTR1, const char **oid, size_t *olen ) \ int FN_NAME( ATTR1_TYPE ATTR1, ATTR2_TYPE ATTR2, const char **oid , \ size_t *olen ) \ { \ - const TYPE_T *cur = LIST; \ + const TYPE_T *cur = (LIST); \ while( cur->descriptor.asn1 != NULL ) { \ - if( cur->ATTR1 == ATTR1 && cur->ATTR2 == ATTR2 ) { \ + if( cur->ATTR1 == (ATTR1) && cur->ATTR2 == (ATTR2) ) { \ *oid = cur->descriptor.asn1; \ *olen = cur->descriptor.asn1_len; \ return( 0 ); \ diff --git a/thirdparty/mbedtls/library/poly1305.c b/thirdparty/mbedtls/library/poly1305.c index b2741191811..2b56c5f7efe 100644 --- a/thirdparty/mbedtls/library/poly1305.c +++ b/thirdparty/mbedtls/library/poly1305.c @@ -58,10 +58,10 @@ #define POLY1305_BLOCK_SIZE_BYTES ( 16U ) #define BYTES_TO_U32_LE( data, offset ) \ - ( (uint32_t) data[offset] \ - | (uint32_t) ( (uint32_t) data[( offset ) + 1] << 8 ) \ - | (uint32_t) ( (uint32_t) data[( offset ) + 2] << 16 ) \ - | (uint32_t) ( (uint32_t) data[( offset ) + 3] << 24 ) \ + ( (uint32_t) (data)[offset] \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 1] << 8 ) \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 2] << 16 ) \ + | (uint32_t) ( (uint32_t) (data)[( offset ) + 3] << 24 ) \ ) /* diff --git a/thirdparty/mbedtls/library/ripemd160.c b/thirdparty/mbedtls/library/ripemd160.c index bd25ada62cd..0791ae4cc93 100644 --- a/thirdparty/mbedtls/library/ripemd160.c +++ b/thirdparty/mbedtls/library/ripemd160.c @@ -147,22 +147,29 @@ int mbedtls_internal_ripemd160_process( mbedtls_ripemd160_context *ctx, D = Dp = ctx->state[3]; E = Ep = ctx->state[4]; -#define F1( x, y, z ) ( x ^ y ^ z ) -#define F2( x, y, z ) ( ( x & y ) | ( ~x & z ) ) -#define F3( x, y, z ) ( ( x | ~y ) ^ z ) -#define F4( x, y, z ) ( ( x & z ) | ( y & ~z ) ) -#define F5( x, y, z ) ( x ^ ( y | ~z ) ) +#define F1( x, y, z ) ( (x) ^ (y) ^ (z) ) +#define F2( x, y, z ) ( ( (x) & (y) ) | ( ~(x) & (z) ) ) +#define F3( x, y, z ) ( ( (x) | ~(y) ) ^ (z) ) +#define F4( x, y, z ) ( ( (x) & (z) ) | ( (y) & ~(z) ) ) +#define F5( x, y, z ) ( (x) ^ ( (y) | ~(z) ) ) -#define S( x, n ) ( ( x << n ) | ( x >> (32 - n) ) ) +#define S( x, n ) ( ( (x) << (n) ) | ( (x) >> (32 - (n)) ) ) -#define P( a, b, c, d, e, r, s, f, k ) \ - a += f( b, c, d ) + X[r] + k; \ - a = S( a, s ) + e; \ - c = S( c, 10 ); +#define P( a, b, c, d, e, r, s, f, k ) \ + do \ + { \ + (a) += f( (b), (c), (d) ) + X[r] + (k); \ + (a) = S( (a), (s) ) + (e); \ + (c) = S( (c), 10 ); \ + } while( 0 ) -#define P2( a, b, c, d, e, r, s, rp, sp ) \ - P( a, b, c, d, e, r, s, F, K ); \ - P( a ## p, b ## p, c ## p, d ## p, e ## p, rp, sp, Fp, Kp ); +#define P2( a, b, c, d, e, r, s, rp, sp ) \ + do \ + { \ + P( (a), (b), (c), (d), (e), (r), (s), F, K ); \ + P( a ## p, b ## p, c ## p, d ## p, e ## p, \ + (rp), (sp), Fp, Kp ); \ + } while( 0 ) #define F F1 #define K 0x00000000 diff --git a/thirdparty/mbedtls/library/sha1.c b/thirdparty/mbedtls/library/sha1.c index e8d4096fbb6..355c83d2f78 100644 --- a/thirdparty/mbedtls/library/sha1.c +++ b/thirdparty/mbedtls/library/sha1.c @@ -152,19 +152,21 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, GET_UINT32_BE( W[14], data, 56 ); GET_UINT32_BE( W[15], data, 60 ); -#define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) +#define S(x,n) (((x) << (n)) | (((x) & 0xFFFFFFFF) >> (32 - (n)))) -#define R(t) \ -( \ - temp = W[( t - 3 ) & 0x0F] ^ W[( t - 8 ) & 0x0F] ^ \ - W[( t - 14 ) & 0x0F] ^ W[ t & 0x0F], \ - ( W[t & 0x0F] = S(temp,1) ) \ -) +#define R(t) \ + ( \ + temp = W[( (t) - 3 ) & 0x0F] ^ W[( (t) - 8 ) & 0x0F] ^ \ + W[( (t) - 14 ) & 0x0F] ^ W[ (t) & 0x0F], \ + ( W[(t) & 0x0F] = S(temp,1) ) \ + ) -#define P(a,b,c,d,e,x) \ -{ \ - e += S(a,5) + F(b,c,d) + K + x; b = S(b,30); \ -} +#define P(a,b,c,d,e,x) \ + do \ + { \ + (e) += S((a),5) + F((b),(c),(d)) + K + (x); \ + (b) = S((b),30); \ + } while( 0 ) A = ctx->state[0]; B = ctx->state[1]; @@ -172,7 +174,7 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, D = ctx->state[3]; E = ctx->state[4]; -#define F(x,y,z) (z ^ (x & (y ^ z))) +#define F(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) #define K 0x5A827999 P( A, B, C, D, E, W[0] ); @@ -199,7 +201,7 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, #undef K #undef F -#define F(x,y,z) (x ^ y ^ z) +#define F(x,y,z) ((x) ^ (y) ^ (z)) #define K 0x6ED9EBA1 P( A, B, C, D, E, R(20) ); @@ -226,7 +228,7 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, #undef K #undef F -#define F(x,y,z) ((x & y) | (z & (x | y))) +#define F(x,y,z) (((x) & (y)) | ((z) & ((x) | (y)))) #define K 0x8F1BBCDC P( A, B, C, D, E, R(40) ); @@ -253,7 +255,7 @@ int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, #undef K #undef F -#define F(x,y,z) (x ^ y ^ z) +#define F(x,y,z) ((x) ^ (y) ^ (z)) #define K 0xCA62C1D6 P( A, B, C, D, E, R(60) ); diff --git a/thirdparty/mbedtls/library/sha256.c b/thirdparty/mbedtls/library/sha256.c index 8a540adfbed..2dc0e1a2c9c 100644 --- a/thirdparty/mbedtls/library/sha256.c +++ b/thirdparty/mbedtls/library/sha256.c @@ -172,8 +172,8 @@ static const uint32_t K[] = 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, }; -#define SHR(x,n) ((x & 0xFFFFFFFF) >> n) -#define ROTR(x,n) (SHR(x,n) | (x << (32 - n))) +#define SHR(x,n) (((x) & 0xFFFFFFFF) >> (n)) +#define ROTR(x,n) (SHR(x,n) | ((x) << (32 - (n)))) #define S0(x) (ROTR(x, 7) ^ ROTR(x,18) ^ SHR(x, 3)) #define S1(x) (ROTR(x,17) ^ ROTR(x,19) ^ SHR(x,10)) @@ -181,21 +181,22 @@ static const uint32_t K[] = #define S2(x) (ROTR(x, 2) ^ ROTR(x,13) ^ ROTR(x,22)) #define S3(x) (ROTR(x, 6) ^ ROTR(x,11) ^ ROTR(x,25)) -#define F0(x,y,z) ((x & y) | (z & (x | y))) -#define F1(x,y,z) (z ^ (x & (y ^ z))) +#define F0(x,y,z) (((x) & (y)) | ((z) & ((x) | (y)))) +#define F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) #define R(t) \ -( \ - W[t] = S1(W[t - 2]) + W[t - 7] + \ - S0(W[t - 15]) + W[t - 16] \ -) + ( \ + W[t] = S1(W[(t) - 2]) + W[(t) - 7] + \ + S0(W[(t) - 15]) + W[(t) - 16] \ + ) -#define P(a,b,c,d,e,f,g,h,x,K) \ -{ \ - temp1 = h + S3(e) + F1(e,f,g) + K + x; \ - temp2 = S2(a) + F0(a,b,c); \ - d += temp1; h = temp1 + temp2; \ -} +#define P(a,b,c,d,e,f,g,h,x,K) \ + do \ + { \ + temp1 = (h) + S3(e) + F1((e),(f),(g)) + (K) + (x); \ + temp2 = S2(a) + F0((a),(b),(c)); \ + (d) += temp1; (h) = temp1 + temp2; \ + } while( 0 ) int mbedtls_internal_sha256_process( mbedtls_sha256_context *ctx, const unsigned char data[64] ) diff --git a/thirdparty/mbedtls/library/sha512.c b/thirdparty/mbedtls/library/sha512.c index 941ecda762c..bdd20b284a8 100644 --- a/thirdparty/mbedtls/library/sha512.c +++ b/thirdparty/mbedtls/library/sha512.c @@ -224,8 +224,8 @@ int mbedtls_internal_sha512_process( mbedtls_sha512_context *ctx, SHA512_VALIDATE_RET( ctx != NULL ); SHA512_VALIDATE_RET( (const unsigned char *)data != NULL ); -#define SHR(x,n) (x >> n) -#define ROTR(x,n) (SHR(x,n) | (x << (64 - n))) +#define SHR(x,n) ((x) >> (n)) +#define ROTR(x,n) (SHR((x),(n)) | ((x) << (64 - (n)))) #define S0(x) (ROTR(x, 1) ^ ROTR(x, 8) ^ SHR(x, 7)) #define S1(x) (ROTR(x,19) ^ ROTR(x,61) ^ SHR(x, 6)) @@ -233,15 +233,16 @@ int mbedtls_internal_sha512_process( mbedtls_sha512_context *ctx, #define S2(x) (ROTR(x,28) ^ ROTR(x,34) ^ ROTR(x,39)) #define S3(x) (ROTR(x,14) ^ ROTR(x,18) ^ ROTR(x,41)) -#define F0(x,y,z) ((x & y) | (z & (x | y))) -#define F1(x,y,z) (z ^ (x & (y ^ z))) +#define F0(x,y,z) (((x) & (y)) | ((z) & ((x) | (y)))) +#define F1(x,y,z) ((z) ^ ((x) & ((y) ^ (z)))) -#define P(a,b,c,d,e,f,g,h,x,K) \ -{ \ - temp1 = h + S3(e) + F1(e,f,g) + K + x; \ - temp2 = S2(a) + F0(a,b,c); \ - d += temp1; h = temp1 + temp2; \ -} +#define P(a,b,c,d,e,f,g,h,x,K) \ + do \ + { \ + temp1 = (h) + S3(e) + F1((e),(f),(g)) + (K) + (x); \ + temp2 = S2(a) + F0((a),(b),(c)); \ + (d) += temp1; (h) = temp1 + temp2; \ + } while( 0 ) for( i = 0; i < 16; i++ ) { diff --git a/thirdparty/mbedtls/library/ssl_ciphersuites.c b/thirdparty/mbedtls/library/ssl_ciphersuites.c index 745474effe4..518f7dde00f 100644 --- a/thirdparty/mbedtls/library/ssl_ciphersuites.c +++ b/thirdparty/mbedtls/library/ssl_ciphersuites.c @@ -43,11 +43,11 @@ /* * Ordered from most preferred to least preferred in terms of security. * - * Current rule (except rc4, weak and null which come last): + * Current rule (except RC4 and 3DES, weak and null which come last): * 1. By key exchange: * Forward-secure non-PSK > forward-secure PSK > ECJPAKE > other non-PSK > other PSK * 2. By key length and cipher: - * ChaCha > AES-256 > Camellia-256 > ARIA-256 > AES-128 > Camellia-128 > ARIA-128 > 3DES + * ChaCha > AES-256 > Camellia-256 > ARIA-256 > AES-128 > Camellia-128 > ARIA-128 * 3. By cipher mode when relevant GCM > CCM > CBC > CCM_8 * 4. By hash function used when relevant * 5. By key exchange/auth again: EC > non-EC @@ -126,11 +126,6 @@ static const int ciphersuite_preference[] = MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256, MBEDTLS_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256, - /* All remaining >= 128-bit ephemeral suites */ - MBEDTLS_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA, - MBEDTLS_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - MBEDTLS_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, - /* The PSK ephemeral suites */ MBEDTLS_TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256, MBEDTLS_TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256, @@ -162,9 +157,6 @@ static const int ciphersuite_preference[] = MBEDTLS_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256, MBEDTLS_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256, - MBEDTLS_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, - MBEDTLS_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, - /* The ECJPAKE suite */ MBEDTLS_TLS_ECJPAKE_WITH_AES_128_CCM_8, @@ -228,11 +220,6 @@ static const int ciphersuite_preference[] = MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256, MBEDTLS_TLS_RSA_WITH_ARIA_128_CBC_SHA256, - /* All remaining >= 128-bit suites */ - MBEDTLS_TLS_RSA_WITH_3DES_EDE_CBC_SHA, - MBEDTLS_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA, - MBEDTLS_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA, - /* The RSA PSK suites */ MBEDTLS_TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256, MBEDTLS_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384, @@ -251,8 +238,6 @@ static const int ciphersuite_preference[] = MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256, MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256, - MBEDTLS_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, - /* The PSK suites */ MBEDTLS_TLS_PSK_WITH_CHACHA20_POLY1305_SHA256, MBEDTLS_TLS_PSK_WITH_AES_256_GCM_SHA384, @@ -275,6 +260,16 @@ static const int ciphersuite_preference[] = MBEDTLS_TLS_PSK_WITH_ARIA_128_GCM_SHA256, MBEDTLS_TLS_PSK_WITH_ARIA_128_CBC_SHA256, + /* 3DES suites */ + MBEDTLS_TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_RSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA, + MBEDTLS_TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA, MBEDTLS_TLS_PSK_WITH_3DES_EDE_CBC_SHA, /* RC4 suites */ @@ -2187,6 +2182,26 @@ const int *mbedtls_ssl_list_ciphersuites( void ) static int supported_ciphersuites[MAX_CIPHERSUITES]; static int supported_init = 0; +static int ciphersuite_is_removed( const mbedtls_ssl_ciphersuite_t *cs_info ) +{ + (void)cs_info; + +#if defined(MBEDTLS_REMOVE_ARC4_CIPHERSUITES) + if( cs_info->cipher == MBEDTLS_CIPHER_ARC4_128 ) + return( 1 ); +#endif /* MBEDTLS_REMOVE_ARC4_CIPHERSUITES */ + +#if defined(MBEDTLS_REMOVE_3DES_CIPHERSUITES) + if( cs_info->cipher == MBEDTLS_CIPHER_DES_EDE3_ECB || + cs_info->cipher == MBEDTLS_CIPHER_DES_EDE3_CBC ) + { + return( 1 ); + } +#endif /* MBEDTLS_REMOVE_3DES_CIPHERSUITES */ + + return( 0 ); +} + const int *mbedtls_ssl_list_ciphersuites( void ) { /* @@ -2202,14 +2217,12 @@ const int *mbedtls_ssl_list_ciphersuites( void ) *p != 0 && q < supported_ciphersuites + MAX_CIPHERSUITES - 1; p++ ) { -#if defined(MBEDTLS_REMOVE_ARC4_CIPHERSUITES) const mbedtls_ssl_ciphersuite_t *cs_info; if( ( cs_info = mbedtls_ssl_ciphersuite_from_id( *p ) ) != NULL && - cs_info->cipher != MBEDTLS_CIPHER_ARC4_128 ) -#else - if( mbedtls_ssl_ciphersuite_from_id( *p ) != NULL ) -#endif + !ciphersuite_is_removed( cs_info ) ) + { *(q++) = *p; + } } *q = 0; diff --git a/thirdparty/mbedtls/library/version_features.c b/thirdparty/mbedtls/library/version_features.c index 4c36d3caaaf..24143d052c3 100644 --- a/thirdparty/mbedtls/library/version_features.c +++ b/thirdparty/mbedtls/library/version_features.c @@ -300,6 +300,9 @@ static const char *features[] = { #if defined(MBEDTLS_REMOVE_ARC4_CIPHERSUITES) "MBEDTLS_REMOVE_ARC4_CIPHERSUITES", #endif /* MBEDTLS_REMOVE_ARC4_CIPHERSUITES */ +#if defined(MBEDTLS_REMOVE_3DES_CIPHERSUITES) + "MBEDTLS_REMOVE_3DES_CIPHERSUITES", +#endif /* MBEDTLS_REMOVE_3DES_CIPHERSUITES */ #if defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) "MBEDTLS_ECP_DP_SECP192R1_ENABLED", #endif /* MBEDTLS_ECP_DP_SECP192R1_ENABLED */ diff --git a/thirdparty/mbedtls/library/x509.c b/thirdparty/mbedtls/library/x509.c index 52b5b649f70..a562df7ca3a 100644 --- a/thirdparty/mbedtls/library/x509.c +++ b/thirdparty/mbedtls/library/x509.c @@ -67,8 +67,15 @@ #include #endif -#define CHECK(code) if( ( ret = code ) != 0 ){ return( ret ); } -#define CHECK_RANGE(min, max, val) if( val < min || val > max ){ return( ret ); } +#define CHECK(code) if( ( ret = ( code ) ) != 0 ){ return( ret ); } +#define CHECK_RANGE(min, max, val) \ + do \ + { \ + if( ( val ) < ( min ) || ( val ) > ( max ) ) \ + { \ + return( ret ); \ + } \ + } while( 0 ) /* * CertificateSerialNumber ::= INTEGER @@ -354,6 +361,8 @@ static int x509_get_attr_type_value( unsigned char **p, MBEDTLS_ASN1_CONSTRUCTED | MBEDTLS_ASN1_SEQUENCE ) ) != 0 ) return( MBEDTLS_ERR_X509_INVALID_NAME + ret ); + end = *p + len; + if( ( end - *p ) < 1 ) return( MBEDTLS_ERR_X509_INVALID_NAME + MBEDTLS_ERR_ASN1_OUT_OF_DATA ); @@ -387,6 +396,12 @@ static int x509_get_attr_type_value( unsigned char **p, val->p = *p; *p += val->len; + if( *p != end ) + { + return( MBEDTLS_ERR_X509_INVALID_NAME + + MBEDTLS_ERR_ASN1_LENGTH_MISMATCH ); + } + cur->next = NULL; return( 0 ); @@ -693,30 +708,25 @@ int mbedtls_x509_get_sig_alg( const mbedtls_x509_buf *sig_oid, const mbedtls_x50 * be either manually updated or extensions should be parsed!) */ int mbedtls_x509_get_ext( unsigned char **p, const unsigned char *end, - mbedtls_x509_buf *ext, int tag ) + mbedtls_x509_buf *ext, int tag ) { int ret; size_t len; - if( *p == end ) - return( 0 ); + /* Extension structure use EXPLICIT tagging. That is, the actual + * `Extensions` structure is wrapped by a tag-length pair using + * the respective context-specific tag. */ + ret = mbedtls_asn1_get_tag( p, end, &ext->len, + MBEDTLS_ASN1_CONTEXT_SPECIFIC | MBEDTLS_ASN1_CONSTRUCTED | tag ); + if( ret != 0 ) + return( MBEDTLS_ERR_X509_INVALID_EXTENSIONS + ret ); - ext->tag = **p; - - if( ( ret = mbedtls_asn1_get_tag( p, end, &ext->len, - MBEDTLS_ASN1_CONTEXT_SPECIFIC | MBEDTLS_ASN1_CONSTRUCTED | tag ) ) != 0 ) - return( ret ); - - ext->p = *p; - end = *p + ext->len; + ext->tag = MBEDTLS_ASN1_CONTEXT_SPECIFIC | MBEDTLS_ASN1_CONSTRUCTED | tag; + ext->p = *p; + end = *p + ext->len; /* * Extensions ::= SEQUENCE SIZE (1..MAX) OF Extension - * - * Extension ::= SEQUENCE { - * extnID OBJECT IDENTIFIER, - * critical BOOLEAN DEFAULT FALSE, - * extnValue OCTET STRING } */ if( ( ret = mbedtls_asn1_get_tag( p, end, &len, MBEDTLS_ASN1_CONSTRUCTED | MBEDTLS_ASN1_SEQUENCE ) ) != 0 ) @@ -1001,8 +1011,8 @@ int mbedtls_x509_time_is_future( const mbedtls_x509_time *from ) */ int mbedtls_x509_self_test( int verbose ) { + int ret = 0; #if defined(MBEDTLS_CERTS_C) && defined(MBEDTLS_SHA256_C) - int ret; uint32_t flags; mbedtls_x509_crt cacert; mbedtls_x509_crt clicert; @@ -1010,6 +1020,7 @@ int mbedtls_x509_self_test( int verbose ) if( verbose != 0 ) mbedtls_printf( " X.509 certificate load: " ); + mbedtls_x509_crt_init( &cacert ); mbedtls_x509_crt_init( &clicert ); ret = mbedtls_x509_crt_parse( &clicert, (const unsigned char *) mbedtls_test_cli_crt, @@ -1019,11 +1030,9 @@ int mbedtls_x509_self_test( int verbose ) if( verbose != 0 ) mbedtls_printf( "failed\n" ); - return( ret ); + goto cleanup; } - mbedtls_x509_crt_init( &cacert ); - ret = mbedtls_x509_crt_parse( &cacert, (const unsigned char *) mbedtls_test_ca_crt, mbedtls_test_ca_crt_len ); if( ret != 0 ) @@ -1031,7 +1040,7 @@ int mbedtls_x509_self_test( int verbose ) if( verbose != 0 ) mbedtls_printf( "failed\n" ); - return( ret ); + goto cleanup; } if( verbose != 0 ) @@ -1043,20 +1052,19 @@ int mbedtls_x509_self_test( int verbose ) if( verbose != 0 ) mbedtls_printf( "failed\n" ); - return( ret ); + goto cleanup; } if( verbose != 0 ) mbedtls_printf( "passed\n\n"); +cleanup: mbedtls_x509_crt_free( &cacert ); mbedtls_x509_crt_free( &clicert ); - - return( 0 ); #else ((void) verbose); - return( 0 ); #endif /* MBEDTLS_CERTS_C && MBEDTLS_SHA1_C */ + return( ret ); } #endif /* MBEDTLS_SELF_TEST */ diff --git a/thirdparty/mbedtls/library/x509_crl.c b/thirdparty/mbedtls/library/x509_crl.c index 8450f87e033..00f8545d7cd 100644 --- a/thirdparty/mbedtls/library/x509_crl.c +++ b/thirdparty/mbedtls/library/x509_crl.c @@ -103,17 +103,17 @@ static int x509_get_crl_ext( unsigned char **p, { int ret; + if( *p == end ) + return( 0 ); + /* * crlExtensions [0] EXPLICIT Extensions OPTIONAL * -- if present, version MUST be v2 */ if( ( ret = mbedtls_x509_get_ext( p, end, ext, 0 ) ) != 0 ) - { - if( ret == MBEDTLS_ERR_ASN1_UNEXPECTED_TAG ) - return( 0 ); - return( ret ); - } + + end = ext->p + ext->len; while( *p < end ) { diff --git a/thirdparty/mbedtls/library/x509_crt.c b/thirdparty/mbedtls/library/x509_crt.c index 35a134950ec..97e1d72e3c7 100644 --- a/thirdparty/mbedtls/library/x509_crt.c +++ b/thirdparty/mbedtls/library/x509_crt.c @@ -65,19 +65,6 @@ #if defined(_WIN32) && !defined(EFIX64) && !defined(EFI32) #include -#if defined(_MSC_VER) && _MSC_VER <= 1600 -/* Visual Studio 2010 and earlier issue a warning when both and - * are included, as they redefine a number of _MAX constants. - * These constants are guaranteed to be the same, though, so we suppress the - * warning when including intsafe.h. - */ -#pragma warning( push ) -#pragma warning( disable : 4005 ) -#endif -#include -#if defined(_MSC_VER) && _MSC_VER <= 1600 -#pragma warning( pop ) -#endif #else #include #endif @@ -381,7 +368,7 @@ static void x509_crt_verify_chain_reset( for( i = 0; i < MBEDTLS_X509_MAX_VERIFY_CHAIN_SIZE; i++ ) { ver_chain->items[i].crt = NULL; - ver_chain->items[i].flags = -1; + ver_chain->items[i].flags = (uint32_t) -1; } ver_chain->len = 0; @@ -406,7 +393,7 @@ static int x509_get_version( unsigned char **p, return( 0 ); } - return( ret ); + return( MBEDTLS_ERR_X509_INVALID_FORMAT + ret ); } end = *p + len; @@ -473,7 +460,7 @@ static int x509_get_uid( unsigned char **p, if( ret == MBEDTLS_ERR_ASN1_UNEXPECTED_TAG ) return( 0 ); - return( ret ); + return( MBEDTLS_ERR_X509_INVALID_FORMAT + ret ); } uid->p = *p; @@ -712,14 +699,13 @@ static int x509_get_crt_ext( unsigned char **p, size_t len; unsigned char *end_ext_data, *end_ext_octet; + if( *p == end ) + return( 0 ); + if( ( ret = mbedtls_x509_get_ext( p, end, &crt->v3_ext, 3 ) ) != 0 ) - { - if( ret == MBEDTLS_ERR_ASN1_UNEXPECTED_TAG ) - return( 0 ); - return( ret ); - } + end = crt->v3_ext.p + crt->v3_ext.len; while( *p < end ) { /* @@ -1291,7 +1277,6 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) char filename[MAX_PATH]; char *p; size_t len = strlen( path ); - int lengthAsInt = 0; WIN32_FIND_DATAW file_data; HANDLE hFind; @@ -1306,18 +1291,7 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) p = filename + len; filename[len++] = '*'; - if ( FAILED ( SizeTToInt( len, &lengthAsInt ) ) ) - return( MBEDTLS_ERR_X509_FILE_IO_ERROR ); - - /* - * Note this function uses the code page CP_ACP, and assumes the incoming - * string is encoded in ANSI, before translating it into Unicode. If the - * incoming string were changed to be UTF-8, then the length check needs to - * change to check the number of characters, not the number of bytes, in the - * incoming string are less than MAX_PATH to avoid a buffer overrun with - * MultiByteToWideChar(). - */ - w_ret = MultiByteToWideChar( CP_ACP, 0, filename, lengthAsInt, szDir, + w_ret = MultiByteToWideChar( CP_ACP, 0, filename, (int)len, szDir, MAX_PATH - 3 ); if( w_ret == 0 ) return( MBEDTLS_ERR_X509_BAD_INPUT_DATA ); @@ -1334,11 +1308,8 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) if( file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) continue; - if ( FAILED( SizeTToInt( wcslen( file_data.cFileName ), &lengthAsInt ) ) ) - return( MBEDTLS_ERR_X509_FILE_IO_ERROR ); - w_ret = WideCharToMultiByte( CP_ACP, 0, file_data.cFileName, - lengthAsInt, + lstrlenW( file_data.cFileName ), p, (int) len - 1, NULL, NULL ); if( w_ret == 0 ) @@ -1467,7 +1438,7 @@ static int x509_info_subject_alt_name( char **buf, size_t *size, } #define CERT_TYPE(type,name) \ - if( ns_cert_type & type ) \ + if( ns_cert_type & (type) ) \ PRINT_ITEM( name ); static int x509_info_cert_type( char **buf, size_t *size, @@ -1494,7 +1465,7 @@ static int x509_info_cert_type( char **buf, size_t *size, } #define KEY_USAGE(code,name) \ - if( key_usage & code ) \ + if( key_usage & (code) ) \ PRINT_ITEM( name ); static int x509_info_key_usage( char **buf, size_t *size, diff --git a/thirdparty/mbedtls/library/x509_csr.c b/thirdparty/mbedtls/library/x509_csr.c index f84425728a4..c8c08c87b23 100644 --- a/thirdparty/mbedtls/library/x509_csr.c +++ b/thirdparty/mbedtls/library/x509_csr.c @@ -279,15 +279,24 @@ int mbedtls_x509_csr_parse( mbedtls_x509_csr *csr, const unsigned char *buf, siz { mbedtls_pem_init( &pem ); ret = mbedtls_pem_read_buffer( &pem, - "-----BEGIN CERTIFICATE REQUEST-----", - "-----END CERTIFICATE REQUEST-----", - buf, NULL, 0, &use_len ); + "-----BEGIN CERTIFICATE REQUEST-----", + "-----END CERTIFICATE REQUEST-----", + buf, NULL, 0, &use_len ); + if( ret == MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT ) + { + ret = mbedtls_pem_read_buffer( &pem, + "-----BEGIN NEW CERTIFICATE REQUEST-----", + "-----END NEW CERTIFICATE REQUEST-----", + buf, NULL, 0, &use_len ); + } if( ret == 0 ) + { /* * Was PEM encoded, parse the result */ ret = mbedtls_x509_csr_parse_der( csr, pem.buf, pem.buflen ); + } mbedtls_pem_free( &pem ); if( ret != MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT ) diff --git a/thirdparty/mbedtls/library/x509write_crt.c b/thirdparty/mbedtls/library/x509write_crt.c index b1ef216c95e..10497e752bb 100644 --- a/thirdparty/mbedtls/library/x509write_crt.c +++ b/thirdparty/mbedtls/library/x509write_crt.c @@ -218,26 +218,51 @@ int mbedtls_x509write_crt_set_authority_key_identifier( mbedtls_x509write_cert * } #endif /* MBEDTLS_SHA1_C */ +static size_t crt_get_unused_bits_for_named_bitstring( unsigned char bitstring, + size_t bit_offset ) +{ + size_t unused_bits; + + /* Count the unused bits removing trailing 0s */ + for( unused_bits = bit_offset; unused_bits < 8; unused_bits++ ) + if( ( ( bitstring >> unused_bits ) & 0x1 ) != 0 ) + break; + + return( unused_bits ); +} + int mbedtls_x509write_crt_set_key_usage( mbedtls_x509write_cert *ctx, unsigned int key_usage ) { unsigned char buf[4], ku; unsigned char *c; int ret; + size_t unused_bits; + const unsigned int allowed_bits = MBEDTLS_X509_KU_DIGITAL_SIGNATURE | + MBEDTLS_X509_KU_NON_REPUDIATION | + MBEDTLS_X509_KU_KEY_ENCIPHERMENT | + MBEDTLS_X509_KU_DATA_ENCIPHERMENT | + MBEDTLS_X509_KU_KEY_AGREEMENT | + MBEDTLS_X509_KU_KEY_CERT_SIGN | + MBEDTLS_X509_KU_CRL_SIGN; - /* We currently only support 7 bits, from 0x80 to 0x02 */ - if( ( key_usage & ~0xfe ) != 0 ) + /* Check that nothing other than the allowed flags is set */ + if( ( key_usage & ~allowed_bits ) != 0 ) return( MBEDTLS_ERR_X509_FEATURE_UNAVAILABLE ); c = buf + 4; - ku = (unsigned char) key_usage; + ku = (unsigned char)key_usage; + unused_bits = crt_get_unused_bits_for_named_bitstring( ku, 1 ); + ret = mbedtls_asn1_write_bitstring( &c, buf, &ku, 8 - unused_bits ); - if( ( ret = mbedtls_asn1_write_bitstring( &c, buf, &ku, 7 ) ) != 4 ) + if( ret < 0 ) return( ret ); + else if( ret < 3 || ret > 4 ) + return( MBEDTLS_ERR_X509_INVALID_FORMAT ); ret = mbedtls_x509write_crt_set_extension( ctx, MBEDTLS_OID_KEY_USAGE, MBEDTLS_OID_SIZE( MBEDTLS_OID_KEY_USAGE ), - 1, buf, 4 ); + 1, c, (size_t)ret ); if( ret != 0 ) return( ret ); @@ -249,16 +274,22 @@ int mbedtls_x509write_crt_set_ns_cert_type( mbedtls_x509write_cert *ctx, { unsigned char buf[4]; unsigned char *c; + size_t unused_bits; int ret; c = buf + 4; - if( ( ret = mbedtls_asn1_write_bitstring( &c, buf, &ns_cert_type, 8 ) ) != 4 ) + unused_bits = crt_get_unused_bits_for_named_bitstring( ns_cert_type, 0 ); + ret = mbedtls_asn1_write_bitstring( &c, + buf, + &ns_cert_type, + 8 - unused_bits ); + if( ret < 3 || ret > 4 ) return( ret ); ret = mbedtls_x509write_crt_set_extension( ctx, MBEDTLS_OID_NS_CERT_TYPE, MBEDTLS_OID_SIZE( MBEDTLS_OID_NS_CERT_TYPE ), - 0, buf, 4 ); + 0, c, (size_t)ret ); if( ret != 0 ) return( ret ); diff --git a/thirdparty/mbedtls/library/x509write_csr.c b/thirdparty/mbedtls/library/x509write_csr.c index 66cee56014c..d70ba0ed927 100644 --- a/thirdparty/mbedtls/library/x509write_csr.c +++ b/thirdparty/mbedtls/library/x509write_csr.c @@ -81,20 +81,39 @@ int mbedtls_x509write_csr_set_extension( mbedtls_x509write_csr *ctx, 0, val, val_len ); } +static size_t csr_get_unused_bits_for_named_bitstring( unsigned char bitstring, + size_t bit_offset ) +{ + size_t unused_bits; + + /* Count the unused bits removing trailing 0s */ + for( unused_bits = bit_offset; unused_bits < 8; unused_bits++ ) + if( ( ( bitstring >> unused_bits ) & 0x1 ) != 0 ) + break; + + return( unused_bits ); +} + int mbedtls_x509write_csr_set_key_usage( mbedtls_x509write_csr *ctx, unsigned char key_usage ) { unsigned char buf[4]; unsigned char *c; + size_t unused_bits; int ret; c = buf + 4; - if( ( ret = mbedtls_asn1_write_bitstring( &c, buf, &key_usage, 7 ) ) != 4 ) + unused_bits = csr_get_unused_bits_for_named_bitstring( key_usage, 0 ); + ret = mbedtls_asn1_write_bitstring( &c, buf, &key_usage, 8 - unused_bits ); + + if( ret < 0 ) return( ret ); + else if( ret < 3 || ret > 4 ) + return( MBEDTLS_ERR_X509_INVALID_FORMAT ); ret = mbedtls_x509write_csr_set_extension( ctx, MBEDTLS_OID_KEY_USAGE, MBEDTLS_OID_SIZE( MBEDTLS_OID_KEY_USAGE ), - buf, 4 ); + c, (size_t)ret ); if( ret != 0 ) return( ret ); @@ -106,16 +125,25 @@ int mbedtls_x509write_csr_set_ns_cert_type( mbedtls_x509write_csr *ctx, { unsigned char buf[4]; unsigned char *c; + size_t unused_bits; int ret; c = buf + 4; - if( ( ret = mbedtls_asn1_write_bitstring( &c, buf, &ns_cert_type, 8 ) ) != 4 ) + unused_bits = csr_get_unused_bits_for_named_bitstring( ns_cert_type, 0 ); + ret = mbedtls_asn1_write_bitstring( &c, + buf, + &ns_cert_type, + 8 - unused_bits ); + + if( ret < 0 ) + return( ret ); + else if( ret < 3 || ret > 4 ) return( ret ); ret = mbedtls_x509write_csr_set_extension( ctx, MBEDTLS_OID_NS_CERT_TYPE, MBEDTLS_OID_SIZE( MBEDTLS_OID_NS_CERT_TYPE ), - buf, 4 ); + c, (size_t)ret ); if( ret != 0 ) return( ret ); From 0d2cb78f3e4d98dd420694b6e968dd5481cff109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 11 Jul 2019 10:11:40 +0200 Subject: [PATCH 004/113] glad: Sync with upstream 0.1.31 (cherry picked from commit 2bbcc521b63acc3c380fc98d07adf8a8720edeb7) --- COPYRIGHT.txt | 2 +- thirdparty/README.md | 2 +- thirdparty/glad/KHR/khrplatform.h | 12 ++++++++++-- thirdparty/glad/glad.c | 2 +- thirdparty/glad/glad/glad.h | 10 ++-------- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index ebeb6b447b5..d7536add0be 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -165,7 +165,7 @@ License: FTL Files: ./thirdparty/glad/ Comment: glad -Copyright: 2013-2018, David Herberth +Copyright: 2013-2019, David Herberth License: Expat Files: ./thirdparty/jpeg_compressor/ diff --git a/thirdparty/README.md b/thirdparty/README.md index d1eaec03eef..462f241b5fc 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -131,7 +131,7 @@ Files extracted from upstream source: ## glad - Upstream: https://github.com/Dav1dde/glad -- Version: 0.1.29 +- Version: 0.1.31 - License: MIT The files we package are automatically generated. diff --git a/thirdparty/glad/KHR/khrplatform.h b/thirdparty/glad/KHR/khrplatform.h index 975bbffed67..5b55ea2b981 100644 --- a/thirdparty/glad/KHR/khrplatform.h +++ b/thirdparty/glad/KHR/khrplatform.h @@ -90,12 +90,20 @@ * int arg2) KHRONOS_APIATTRIBUTES; */ +#if defined(__SCITECH_SNAP__) && !defined(KHRONOS_STATIC) +# define KHRONOS_STATIC 1 +#endif + /*------------------------------------------------------------------------- * Definition of KHRONOS_APICALL *------------------------------------------------------------------------- * This precedes the return type of the function in the function prototype. */ -#if defined(_WIN32) && !defined(__SCITECH_SNAP__) +#if defined(KHRONOS_STATIC) + /* If the preprocessor constant KHRONOS_STATIC is defined, make the + * header compatible with static linking. */ +# define KHRONOS_APICALL +#elif defined(_WIN32) # define KHRONOS_APICALL __declspec(dllimport) #elif defined (__SYMBIAN32__) # define KHRONOS_APICALL IMPORT_C @@ -111,7 +119,7 @@ * This follows the return type of the function and precedes the function * name in the function prototype. */ -#if defined(_WIN32) && !defined(_WIN32_WCE) && !defined(__SCITECH_SNAP__) +#if defined(_WIN32) && !defined(_WIN32_WCE) && !defined(KHRONOS_STATIC) /* Win32 but not WinCE */ # define KHRONOS_APIENTRY __stdcall #else diff --git a/thirdparty/glad/glad.c b/thirdparty/glad/glad.c index e7c64a7e179..6bc54a33b06 100644 --- a/thirdparty/glad/glad.c +++ b/thirdparty/glad/glad.c @@ -1,6 +1,6 @@ /* - OpenGL loader generated by glad 0.1.29 on Mon Mar 4 12:47:22 2019. + OpenGL loader generated by glad 0.1.31 on Thu Jul 11 10:09:18 2019. Language/Generator: C/C++ Specification: gl diff --git a/thirdparty/glad/glad/glad.h b/thirdparty/glad/glad/glad.h index 6345de6f7ae..cba90ff9bda 100644 --- a/thirdparty/glad/glad/glad.h +++ b/thirdparty/glad/glad/glad.h @@ -1,6 +1,6 @@ /* - OpenGL loader generated by glad 0.1.29 on Mon Mar 4 12:47:22 2019. + OpenGL loader generated by glad 0.1.31 on Thu Jul 11 10:09:18 2019. Language/Generator: C/C++ Specification: gl @@ -31,13 +31,7 @@ #define __gl_h_ #if defined(_WIN32) && !defined(APIENTRY) && !defined(__CYGWIN__) && !defined(__SCITECH_SNAP__) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN 1 -#endif -#ifndef NOMINMAX -#define NOMINMAX 1 -#endif -#include +#define APIENTRY __stdcall #endif #ifndef APIENTRY From 82773cc7aa020250f2d10e12fa324c992e58518a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 11 Jul 2019 10:16:47 +0200 Subject: [PATCH 005/113] stb_vorbis: Update to upstream version 1.16 (cherry picked from commit 3e727bc659345bd8d4624e53a6256df5f1a43d86) --- COPYRIGHT.txt | 2 +- thirdparty/README.md | 2 +- thirdparty/misc/stb_vorbis.c | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index d7536add0be..08218688895 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -321,7 +321,7 @@ License: BSD-3-clause Files: ./thirdparty/misc/stb_truetype.h ./thirdparty/misc/stb_vorbis.c Comment: stb libraries -Copyright: 2007-2017, Sean Barrett +Copyright: 2007-2019, Sean Barrett License: public-domain Files: ./thirdparty/misc/triangulator.cpp diff --git a/thirdparty/README.md b/thirdparty/README.md index 462f241b5fc..8eb7741282c 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -409,7 +409,7 @@ Collection of single-file libraries used in Godot components. * License: Public Domain (Unlicense) or MIT - `stb_vorbis.c` * Upstream: https://github.com/nothings/stb - * Version: 1.15 + * Version: 1.16 * License: Public Domain (Unlicense) or MIT diff --git a/thirdparty/misc/stb_vorbis.c b/thirdparty/misc/stb_vorbis.c index 88276026efa..71af404daec 100644 --- a/thirdparty/misc/stb_vorbis.c +++ b/thirdparty/misc/stb_vorbis.c @@ -1,4 +1,4 @@ -// Ogg Vorbis audio decoder - v1.15 - public domain +// Ogg Vorbis audio decoder - v1.16 - public domain // http://nothings.org/stb_vorbis/ // // Original version written by Sean Barrett in 2007. @@ -33,6 +33,7 @@ // Timur Gagiev // // Partial history: +// 1.16 - 2019-03-04 - fix warnings // 1.15 - 2019-02-07 - explicit failure if Ogg Skeleton data is found // 1.14 - 2018-02-11 - delete bogus dealloca usage // 1.13 - 2018-01-29 - fix truncation of last frame (hopefully) @@ -4990,7 +4991,13 @@ stb_vorbis * stb_vorbis_open_file(FILE *file, int close_on_free, int *error, con stb_vorbis * stb_vorbis_open_filename(const char *filename, int *error, const stb_vorbis_alloc *alloc) { - FILE *f = fopen(filename, "rb"); + FILE *f; +#if defined(_WIN32) && defined(__STDC_WANT_SECURE_LIB__) + if (0 != fopen_s(&f, filename, "rb")) + f = NULL; +#else + f = fopen(filename, "rb"); +#endif if (f) return stb_vorbis_open_file(f, TRUE, error, alloc); if (error) *error = VORBIS_file_open_failure; From b021671681b9c92435575f6e438f0621e95515d7 Mon Sep 17 00:00:00 2001 From: Jonathan Mannancheril Date: Sat, 9 Nov 2019 23:39:29 -0600 Subject: [PATCH 006/113] Update stb_vorbis to 1.17 (cherry picked from commit 97af5569d90df96378cd9dbec5676063b037e2f1) --- thirdparty/README.md | 2 +- thirdparty/misc/stb_vorbis.c | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index 8eb7741282c..913f011e499 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -409,7 +409,7 @@ Collection of single-file libraries used in Godot components. * License: Public Domain (Unlicense) or MIT - `stb_vorbis.c` * Upstream: https://github.com/nothings/stb - * Version: 1.16 + * Version: 1.17 * License: Public Domain (Unlicense) or MIT diff --git a/thirdparty/misc/stb_vorbis.c b/thirdparty/misc/stb_vorbis.c index 71af404daec..4ab8880d5de 100644 --- a/thirdparty/misc/stb_vorbis.c +++ b/thirdparty/misc/stb_vorbis.c @@ -1,4 +1,4 @@ -// Ogg Vorbis audio decoder - v1.16 - public domain +// Ogg Vorbis audio decoder - v1.17 - public domain // http://nothings.org/stb_vorbis/ // // Original version written by Sean Barrett in 2007. @@ -30,9 +30,10 @@ // Tom Beaumont Ingo Leitgeb Nicolas Guillemot // Phillip Bennefall Rohit Thiago Goulart // manxorist@github saga musix github:infatum -// Timur Gagiev +// Timur Gagiev Maxwell Koo // // Partial history: +// 1.17 - 2019-07-08 - fix CVE-2019-13217..CVE-2019-13223 (by ForAllSecure) // 1.16 - 2019-03-04 - fix warnings // 1.15 - 2019-02-07 - explicit failure if Ogg Skeleton data is found // 1.14 - 2018-02-11 - delete bogus dealloca usage @@ -1202,8 +1203,10 @@ static int lookup1_values(int entries, int dim) int r = (int) floor(exp((float) log((float) entries) / dim)); if ((int) floor(pow((float) r+1, dim)) <= entries) // (int) cast for MinGW warning; ++r; // floor() to avoid _ftol() when non-CRT - assert(pow((float) r+1, dim) > entries); - assert((int) floor(pow((float) r, dim)) <= entries); // (int),floor() as above + if (pow((float) r+1, dim) <= entries) + return -1; + if ((int) floor(pow((float) r, dim)) > entries) + return -1; return r; } @@ -2013,7 +2016,7 @@ static __forceinline void draw_line(float *output, int x0, int y0, int x1, int y ady -= abs(base) * adx; if (x1 > n) x1 = n; if (x < x1) { - LINE_OP(output[x], inverse_db_table[y]); + LINE_OP(output[x], inverse_db_table[y&255]); for (++x; x < x1; ++x) { err += ady; if (err >= adx) { @@ -2021,7 +2024,7 @@ static __forceinline void draw_line(float *output, int x0, int y0, int x1, int y y += sy; } else y += base; - LINE_OP(output[x], inverse_db_table[y]); + LINE_OP(output[x], inverse_db_table[y&255]); } } } @@ -3048,7 +3051,6 @@ static float *get_window(vorb *f, int len) len <<= 1; if (len == f->blocksize_0) return f->window[0]; if (len == f->blocksize_1) return f->window[1]; - assert(0); return NULL; } @@ -3454,6 +3456,7 @@ static int vorbis_finish_frame(stb_vorbis *f, int len, int left, int right) if (f->previous_length) { int i,j, n = f->previous_length; float *w = get_window(f, n); + if (w == NULL) return 0; for (i=0; i < f->channels; ++i) { for (j=0; j < n; ++j) f->channel_buffers[i][left+j] = @@ -3695,6 +3698,7 @@ static int start_decoder(vorb *f) while (current_entry < c->entries) { int limit = c->entries - current_entry; int n = get_bits(f, ilog(limit)); + if (current_length >= 32) return error(f, VORBIS_invalid_setup); if (current_entry + n > (int) c->entries) { return error(f, VORBIS_invalid_setup); } memset(lengths + current_entry, current_length, n); current_entry += n; @@ -3798,7 +3802,9 @@ static int start_decoder(vorb *f) c->value_bits = get_bits(f, 4)+1; c->sequence_p = get_bits(f,1); if (c->lookup_type == 1) { - c->lookup_values = lookup1_values(c->entries, c->dimensions); + int values = lookup1_values(c->entries, c->dimensions); + if (values < 0) return error(f, VORBIS_invalid_setup); + c->lookup_values = (uint32) values; } else { c->lookup_values = c->entries * c->dimensions; } @@ -3934,6 +3940,9 @@ static int start_decoder(vorb *f) p[j].id = j; } qsort(p, g->values, sizeof(p[0]), point_compare); + for (j=0; j < g->values-1; ++j) + if (p[j].x == p[j+1].x) + return error(f, VORBIS_invalid_setup); for (j=0; j < g->values; ++j) g->sorted_order[j] = (uint8) p[j].id; // precompute the neighbors @@ -4020,6 +4029,7 @@ static int start_decoder(vorb *f) max_submaps = m->submaps; if (get_bits(f,1)) { m->coupling_steps = get_bits(f,8)+1; + if (m->coupling_steps > f->channels) return error(f, VORBIS_invalid_setup); for (k=0; k < m->coupling_steps; ++k) { m->chan[k].magnitude = get_bits(f, ilog(f->channels-1)); m->chan[k].angle = get_bits(f, ilog(f->channels-1)); @@ -5386,6 +5396,12 @@ int stb_vorbis_get_samples_float(stb_vorbis *f, int channels, float **buffer, in #endif // STB_VORBIS_NO_PULLDATA_API /* Version history + 1.17 - 2019-07-08 - fix CVE-2019-13217, -13218, -13219, -13220, -13221, -13222, -13223 + found with Mayhem by ForAllSecure + 1.16 - 2019-03-04 - fix warnings + 1.15 - 2019-02-07 - explicit failure if Ogg Skeleton data is found + 1.14 - 2018-02-11 - delete bogus dealloca usage + 1.13 - 2018-01-29 - fix truncation of last frame (hopefully) 1.12 - 2017-11-21 - limit residue begin/end to blocksize/2 to avoid large temp allocs in bad/corrupt files 1.11 - 2017-07-23 - fix MinGW compilation 1.10 - 2017-03-03 - more robust seeking; fix negative ilog(); clear error in open_memory From 8067207257aea19f59fcb0e68474423e00e55c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 11 Jul 2019 10:27:11 +0200 Subject: [PATCH 007/113] pcre2: Update to upstream version 10.33 (cherry picked from commit 518e23424ecdd8411e8569b45b515129c4bbba15) --- COPYRIGHT.txt | 4 +- modules/regex/SCsub | 1 + thirdparty/README.md | 4 +- thirdparty/pcre2/AUTHORS | 6 +- thirdparty/pcre2/LICENCE | 6 +- thirdparty/pcre2/src/config.h | 15 +- thirdparty/pcre2/src/pcre2.h | 104 +- thirdparty/pcre2/src/pcre2_auto_possess.c | 14 +- thirdparty/pcre2/src/pcre2_chartables.c | 18 +- thirdparty/pcre2/src/pcre2_compile.c | 684 +- thirdparty/pcre2/src/pcre2_context.c | 22 +- thirdparty/pcre2/src/pcre2_convert.c | 2 +- thirdparty/pcre2/src/pcre2_dfa_match.c | 71 +- thirdparty/pcre2/src/pcre2_error.c | 19 +- thirdparty/pcre2/src/pcre2_extuni.c | 4 +- thirdparty/pcre2/src/pcre2_internal.h | 249 +- thirdparty/pcre2/src/pcre2_intmodedep.h | 5 +- thirdparty/pcre2/src/pcre2_jit_compile.c | 2384 ++++-- thirdparty/pcre2/src/pcre2_jit_match.c | 6 +- thirdparty/pcre2/src/pcre2_maketables.c | 26 +- thirdparty/pcre2/src/pcre2_match.c | 88 +- thirdparty/pcre2/src/pcre2_match_data.c | 8 +- thirdparty/pcre2/src/pcre2_script_run.c | 441 + thirdparty/pcre2/src/pcre2_study.c | 16 +- thirdparty/pcre2/src/pcre2_substitute.c | 67 +- thirdparty/pcre2/src/pcre2_tables.c | 71 +- thirdparty/pcre2/src/pcre2_ucd.c | 7235 +++++++++-------- thirdparty/pcre2/src/pcre2_ucp.h | 1 + thirdparty/pcre2/src/pcre2_xclass.c | 6 +- .../pcre2/src/sljit/sljitConfigInternal.h | 2 +- .../pcre2/src/sljit/sljitExecAllocator.c | 46 +- thirdparty/pcre2/src/sljit/sljitLir.c | 11 +- .../pcre2/src/sljit/sljitNativeARM_64.c | 20 +- .../pcre2/src/sljit/sljitNativeMIPS_32.c | 11 +- .../pcre2/src/sljit/sljitNativeMIPS_64.c | 13 +- .../pcre2/src/sljit/sljitNativeMIPS_common.c | 164 +- .../pcre2/src/sljit/sljitNativePPC_common.c | 2 +- 37 files changed, 7301 insertions(+), 4545 deletions(-) create mode 100644 thirdparty/pcre2/src/pcre2_script_run.c diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 08218688895..a41a869e2b7 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -350,8 +350,8 @@ License: BSD-3-clause Files: ./thirdparty/pcre2/ Comment: PCRE2 -Copyright: 1997-2018, University of Cambridge, - 2009-2018, Zoltan Herczeg +Copyright: 1997-2019, University of Cambridge, + 2009-2019, Zoltan Herczeg License: BSD-3-clause Files: ./thirdparty/pvrtccompressor/ diff --git a/modules/regex/SCsub b/modules/regex/SCsub index 99c25add458..aaee97d4674 100644 --- a/modules/regex/SCsub +++ b/modules/regex/SCsub @@ -33,6 +33,7 @@ if env['builtin_pcre2']: "pcre2_newline.c", "pcre2_ord2utf.c", "pcre2_pattern_info.c", + "pcre2_script_run.c", "pcre2_serialize.c", "pcre2_string_utils.c", "pcre2_study.c", diff --git a/thirdparty/README.md b/thirdparty/README.md index 913f011e499..66af7720223 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -444,8 +444,8 @@ Files extracted from upstream source: ## pcre2 -- Upstream: http://www.pcre.org/ -- Version: 10.32 +- Upstream: http://www.pcre.org +- Version: 10.33 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/pcre2/AUTHORS b/thirdparty/pcre2/AUTHORS index d5592bbc5b5..8d4e15a247f 100644 --- a/thirdparty/pcre2/AUTHORS +++ b/thirdparty/pcre2/AUTHORS @@ -8,7 +8,7 @@ Email domain: cam.ac.uk University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2018 University of Cambridge +Copyright (c) 1997-2019 University of Cambridge All rights reserved @@ -19,7 +19,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2010-2018 Zoltan Herczeg +Copyright(c) 2010-2019 Zoltan Herczeg All rights reserved. @@ -30,7 +30,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Emain domain: freemail.hu -Copyright(c) 2009-2018 Zoltan Herczeg +Copyright(c) 2009-2019 Zoltan Herczeg All rights reserved. #### diff --git a/thirdparty/pcre2/LICENCE b/thirdparty/pcre2/LICENCE index b0f8804fffa..142b3b3f9ad 100644 --- a/thirdparty/pcre2/LICENCE +++ b/thirdparty/pcre2/LICENCE @@ -26,7 +26,7 @@ Email domain: cam.ac.uk University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2018 University of Cambridge +Copyright (c) 1997-2019 University of Cambridge All rights reserved. @@ -37,7 +37,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Email domain: freemail.hu -Copyright(c) 2010-2018 Zoltan Herczeg +Copyright(c) 2010-2019 Zoltan Herczeg All rights reserved. @@ -48,7 +48,7 @@ Written by: Zoltan Herczeg Email local part: hzmester Email domain: freemail.hu -Copyright(c) 2009-2018 Zoltan Herczeg +Copyright(c) 2009-2019 Zoltan Herczeg All rights reserved. diff --git a/thirdparty/pcre2/src/config.h b/thirdparty/pcre2/src/config.h index 89a52ef8485..25d45eeb388 100644 --- a/thirdparty/pcre2/src/config.h +++ b/thirdparty/pcre2/src/config.h @@ -35,6 +35,10 @@ sure both macros are undefined; an emulation function will then be used. */ */ /* #undef BSR_ANYCRLF */ +/* Define to any value to disable the use of the z and t modifiers in + formatting settings such as %zu or %td (this is rarely needed). */ +/* #undef DISABLE_PERCENT_ZT */ + /* If you are compiling for a system that uses EBCDIC instead of ASCII character codes, define this macro to any value. When EBCDIC is set, PCRE2 assumes that all input strings are in EBCDIC. If you do not define this @@ -214,7 +218,7 @@ sure both macros are undefined; an emulation function will then be used. */ #define PACKAGE_NAME "PCRE2" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PCRE2 10.32" +#define PACKAGE_STRING "PCRE2 10.33" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "pcre2" @@ -223,7 +227,7 @@ sure both macros are undefined; an emulation function will then be used. */ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "10.32" +#define PACKAGE_VERSION "10.33" /* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested parentheses (of any kind) in a pattern. This limits the amount of system @@ -299,6 +303,11 @@ sure both macros are undefined; an emulation function will then be used. */ /* Define to any value to enable callout script support in pcre2grep. */ /* #undef SUPPORT_PCRE2GREP_CALLOUT */ +/* Define to any value to enable fork support in pcre2grep callout scripts. + This will have no effect unless SUPPORT_PCRE2GREP_CALLOUT is also defined. + */ +/* #undef SUPPORT_PCRE2GREP_CALLOUT_FORK */ + /* Define to any value to enable JIT support in pcre2grep. Note that this will have no effect unless SUPPORT_JIT is also defined. */ /* #undef SUPPORT_PCRE2GREP_JIT */ @@ -343,7 +352,7 @@ sure both macros are undefined; an emulation function will then be used. */ #endif /* Version number of package */ -#define VERSION "10.32" +#define VERSION "10.33" /* Define to 1 if on MINIX. */ /* #undef _MINIX */ diff --git a/thirdparty/pcre2/src/pcre2.h b/thirdparty/pcre2/src/pcre2.h index 3d2feb7a6bd..102b5d91f1a 100644 --- a/thirdparty/pcre2/src/pcre2.h +++ b/thirdparty/pcre2/src/pcre2.h @@ -42,15 +42,9 @@ POSSIBILITY OF SUCH DAMAGE. /* The current PCRE version information. */ #define PCRE2_MAJOR 10 -#define PCRE2_MINOR 32 +#define PCRE2_MINOR 33 #define PCRE2_PRERELEASE -#define PCRE2_DATE 2018-09-10 - -/* For the benefit of systems without stdint.h, an alternative is to use -inttypes.h. The existence of these headers is checked by configure or CMake. */ - -#define PCRE2_HAVE_STDINT_H 1 -#define PCRE2_HAVE_INTTYPES_H 1 +#define PCRE2_DATE 2019-04-16 /* When an application links to a PCRE DLL in Windows, the symbols that are imported have to be identified as such. When building PCRE2, the appropriate @@ -87,18 +81,15 @@ set, we ensure here that it has no effect. */ #define PCRE2_CALL_CONVENTION #endif -/* Have to include limits.h, stdlib.h and stdint.h (or inttypes.h) to ensure -that size_t and uint8_t, UCHAR_MAX, etc are defined. If the system has neither -header, the relevant values must be provided by some other means. */ +/* Have to include limits.h, stdlib.h, and inttypes.h to ensure that size_t and +uint8_t, UCHAR_MAX, etc are defined. Some systems that do have inttypes.h do +not have stdint.h, which is why we use inttypes.h, which according to the C +standard is a superset of stdint.h. If none of these headers are available, +the relevant values must be provided by some other means. */ #include #include - -#if PCRE2_HAVE_STDINT_H -#include -#elif PCRE2_HAVE_INTTYPES_H #include -#endif /* Allow for C++ users compiling this directly. */ @@ -158,43 +149,37 @@ D is inspected during pcre2_dfa_match() execution #define PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL 0x00000002u /* C */ #define PCRE2_EXTRA_MATCH_WORD 0x00000004u /* C */ #define PCRE2_EXTRA_MATCH_LINE 0x00000008u /* C */ +#define PCRE2_EXTRA_ESCAPED_CR_IS_LF 0x00000010u /* C */ +#define PCRE2_EXTRA_ALT_BSUX 0x00000020u /* C */ /* These are for pcre2_jit_compile(). */ #define PCRE2_JIT_COMPLETE 0x00000001u /* For full matching */ #define PCRE2_JIT_PARTIAL_SOFT 0x00000002u #define PCRE2_JIT_PARTIAL_HARD 0x00000004u +#define PCRE2_JIT_INVALID_UTF 0x00000100u -/* These are for pcre2_match(), pcre2_dfa_match(), and pcre2_jit_match(). Note -that PCRE2_ANCHORED and PCRE2_NO_UTF_CHECK can also be passed to these -functions (though pcre2_jit_match() ignores the latter since it bypasses all -sanity checks). */ +/* These are for pcre2_match(), pcre2_dfa_match(), pcre2_jit_match(), and +pcre2_substitute(). Some are allowed only for one of the functions, and in +these cases it is noted below. Note that PCRE2_ANCHORED, PCRE2_ENDANCHORED and +PCRE2_NO_UTF_CHECK can also be passed to these functions (though +pcre2_jit_match() ignores the latter since it bypasses all sanity checks). */ -#define PCRE2_NOTBOL 0x00000001u -#define PCRE2_NOTEOL 0x00000002u -#define PCRE2_NOTEMPTY 0x00000004u /* ) These two must be kept */ -#define PCRE2_NOTEMPTY_ATSTART 0x00000008u /* ) adjacent to each other. */ -#define PCRE2_PARTIAL_SOFT 0x00000010u -#define PCRE2_PARTIAL_HARD 0x00000020u - -/* These are additional options for pcre2_dfa_match(). */ - -#define PCRE2_DFA_RESTART 0x00000040u -#define PCRE2_DFA_SHORTEST 0x00000080u - -/* These are additional options for pcre2_substitute(), which passes any others -through to pcre2_match(). */ - -#define PCRE2_SUBSTITUTE_GLOBAL 0x00000100u -#define PCRE2_SUBSTITUTE_EXTENDED 0x00000200u -#define PCRE2_SUBSTITUTE_UNSET_EMPTY 0x00000400u -#define PCRE2_SUBSTITUTE_UNKNOWN_UNSET 0x00000800u -#define PCRE2_SUBSTITUTE_OVERFLOW_LENGTH 0x00001000u - -/* A further option for pcre2_match(), not allowed for pcre2_dfa_match(), -ignored for pcre2_jit_match(). */ - -#define PCRE2_NO_JIT 0x00002000u +#define PCRE2_NOTBOL 0x00000001u +#define PCRE2_NOTEOL 0x00000002u +#define PCRE2_NOTEMPTY 0x00000004u /* ) These two must be kept */ +#define PCRE2_NOTEMPTY_ATSTART 0x00000008u /* ) adjacent to each other. */ +#define PCRE2_PARTIAL_SOFT 0x00000010u +#define PCRE2_PARTIAL_HARD 0x00000020u +#define PCRE2_DFA_RESTART 0x00000040u /* pcre2_dfa_match() only */ +#define PCRE2_DFA_SHORTEST 0x00000080u /* pcre2_dfa_match() only */ +#define PCRE2_SUBSTITUTE_GLOBAL 0x00000100u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_EXTENDED 0x00000200u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_UNSET_EMPTY 0x00000400u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_UNKNOWN_UNSET 0x00000800u /* pcre2_substitute() only */ +#define PCRE2_SUBSTITUTE_OVERFLOW_LENGTH 0x00001000u /* pcre2_substitute() only */ +#define PCRE2_NO_JIT 0x00002000u /* Not for pcre2_dfa_match() */ +#define PCRE2_COPY_MATCHED_SUBJECT 0x00004000u /* Options for pcre2_pattern_convert(). */ @@ -318,6 +303,8 @@ pcre2_pattern_convert(). */ #define PCRE2_ERROR_BAD_LITERAL_OPTIONS 192 #define PCRE2_ERROR_SUPPORTED_ONLY_IN_UNICODE 193 #define PCRE2_ERROR_INVALID_HYPHEN_IN_OPTIONS 194 +#define PCRE2_ERROR_ALPHA_ASSERTION_UNKNOWN 195 +#define PCRE2_ERROR_SCRIPT_RUN_NOT_AVAILABLE 196 /* "Expected" matching error codes: no match and partial match. */ @@ -504,10 +491,10 @@ typedef struct pcre2_real_jit_stack pcre2_jit_stack; \ typedef pcre2_jit_stack *(*pcre2_jit_callback)(void *); -/* The structure for passing out data via the pcre_callout_function. We use a -structure so that new fields can be added on the end in future versions, -without changing the API of the function, thereby allowing old clients to work -without modification. Define the generic version in a macro; the width-specific +/* The structures for passing out data via callout functions. We use structures +so that new fields can be added on the end in future versions, without changing +the API of the function, thereby allowing old clients to work without +modification. Define the generic versions in a macro; the width-specific versions are generated from this macro below. */ /* Flags for the callout_flags field. These are cleared after a callout. */ @@ -549,7 +536,19 @@ typedef struct pcre2_callout_enumerate_block { \ PCRE2_SIZE callout_string_length; /* Length of string compiled into pattern */ \ PCRE2_SPTR callout_string; /* String compiled into pattern */ \ /* ------------------------------------------------------------------ */ \ -} pcre2_callout_enumerate_block; +} pcre2_callout_enumerate_block; \ +\ +typedef struct pcre2_substitute_callout_block { \ + uint32_t version; /* Identifies version of block */ \ + /* ------------------------ Version 0 ------------------------------- */ \ + PCRE2_SPTR input; /* Pointer to input subject string */ \ + PCRE2_SPTR output; /* Pointer to output buffer */ \ + PCRE2_SIZE output_offsets[2]; /* Changed portion of the output */ \ + PCRE2_SIZE *ovector; /* Pointer to current ovector */ \ + uint32_t oveccount; /* Count of pairs set in ovector */ \ + uint32_t subscount; /* Substitution number */ \ + /* ------------------------------------------------------------------ */ \ +} pcre2_substitute_callout_block; /* List the generic forms of all other functions in macros, which will be @@ -604,6 +603,9 @@ PCRE2_EXP_DECL void PCRE2_CALL_CONVENTION \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_callout(pcre2_match_context *, \ int (*)(pcre2_callout_block *, void *), void *); \ +PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ + pcre2_set_substitute_callout(pcre2_match_context *, \ + int (*)(pcre2_substitute_callout_block *, void *), void *); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ pcre2_set_depth_limit(pcre2_match_context *, uint32_t); \ PCRE2_EXP_DECL int PCRE2_CALL_CONVENTION \ @@ -807,6 +809,7 @@ pcre2_compile are called by application code. */ #define pcre2_callout_block PCRE2_SUFFIX(pcre2_callout_block_) #define pcre2_callout_enumerate_block PCRE2_SUFFIX(pcre2_callout_enumerate_block_) +#define pcre2_substitute_callout_block PCRE2_SUFFIX(pcre2_substitute_callout_block_) #define pcre2_general_context PCRE2_SUFFIX(pcre2_general_context_) #define pcre2_compile_context PCRE2_SUFFIX(pcre2_compile_context_) #define pcre2_convert_context PCRE2_SUFFIX(pcre2_convert_context_) @@ -872,6 +875,7 @@ pcre2_compile are called by application code. */ #define pcre2_set_newline PCRE2_SUFFIX(pcre2_set_newline_) #define pcre2_set_parens_nest_limit PCRE2_SUFFIX(pcre2_set_parens_nest_limit_) #define pcre2_set_offset_limit PCRE2_SUFFIX(pcre2_set_offset_limit_) +#define pcre2_set_substitute_callout PCRE2_SUFFIX(pcre2_set_substitute_callout_) #define pcre2_substitute PCRE2_SUFFIX(pcre2_substitute_) #define pcre2_substring_copy_byname PCRE2_SUFFIX(pcre2_substring_copy_byname_) #define pcre2_substring_copy_bynumber PCRE2_SUFFIX(pcre2_substring_copy_bynumber_) diff --git a/thirdparty/pcre2/src/pcre2_auto_possess.c b/thirdparty/pcre2/src/pcre2_auto_possess.c index 2ce152e9522..6d7b7c4a4d9 100644 --- a/thirdparty/pcre2/src/pcre2_auto_possess.c +++ b/thirdparty/pcre2/src/pcre2_auto_possess.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -605,6 +605,15 @@ for(;;) if (cb->had_recurse) return FALSE; break; + /* A script run might have to backtrack if the iterated item can match + characters from more than one script. So give up unless repeating an + explicit character. */ + + case OP_SCRIPT_RUN: + if (base_list[0] != OP_CHAR && base_list[0] != OP_CHARI) + return FALSE; + break; + /* Atomic sub-patterns and assertions can always auto-possessify their last iterator. However, if the group was entered as a result of checking a previous iterator, this is not possible. */ @@ -614,7 +623,6 @@ for(;;) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: - return !entered_a_group; } @@ -1043,7 +1051,7 @@ for(;;) if (chr > 255) break; class_bitset = (uint8_t *) ((list_ptr == list ? code : base_end) - list_ptr[2]); - if ((class_bitset[chr >> 3] & (1 << (chr & 7))) != 0) return FALSE; + if ((class_bitset[chr >> 3] & (1u << (chr & 7))) != 0) return FALSE; break; #ifdef SUPPORT_WIDE_CHARS diff --git a/thirdparty/pcre2/src/pcre2_chartables.c b/thirdparty/pcre2/src/pcre2_chartables.c index 4046500c00d..0e07edb4943 100644 --- a/thirdparty/pcre2/src/pcre2_chartables.c +++ b/thirdparty/pcre2/src/pcre2_chartables.c @@ -157,8 +157,8 @@ graph print, punct, and cntrl. Other classes are built from combinations. */ /* This table identifies various classes of character by individual bits: 0x01 white space character 0x02 letter - 0x04 decimal digit - 0x08 hexadecimal digit + 0x04 lower case letter + 0x08 decimal digit 0x10 alphanumeric or '_' */ @@ -168,16 +168,16 @@ graph print, punct, and cntrl. Other classes are built from combinations. */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - ' */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ( - / */ - 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ - 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x00, /* 8 - ? */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ + 0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, /* 0 - 7 */ + 0x18,0x18,0x00,0x00,0x00,0x00,0x00,0x00, /* 8 - ? */ + 0x00,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* @ - G */ 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ 0x12,0x12,0x12,0x00,0x00,0x00,0x00,0x10, /* X - _ */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ - 0x12,0x12,0x12,0x00,0x00,0x00,0x00,0x00, /* x -127 */ + 0x00,0x16,0x16,0x16,0x16,0x16,0x16,0x16, /* ` - g */ + 0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16, /* h - o */ + 0x16,0x16,0x16,0x16,0x16,0x16,0x16,0x16, /* p - w */ + 0x16,0x16,0x16,0x00,0x00,0x00,0x00,0x00, /* x -127 */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ diff --git a/thirdparty/pcre2/src/pcre2_compile.c b/thirdparty/pcre2/src/pcre2_compile.c index 6bb1de36100..068735ae8ee 100644 --- a/thirdparty/pcre2/src/pcre2_compile.c +++ b/thirdparty/pcre2/src/pcre2_compile.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -240,49 +240,57 @@ code (meta_extra_lengths, just below) must be updated to remain in step. */ #define META_RANGE_LITERAL 0x801f0000u /* range defined literally */ #define META_RECURSE 0x80200000u /* Recursion */ #define META_RECURSE_BYNAME 0x80210000u /* (?&name) */ +#define META_SCRIPT_RUN 0x80220000u /* (*script_run:...) */ /* These must be kept together to make it easy to check that an assertion is present where expected in a conditional group. */ -#define META_LOOKAHEAD 0x80220000u /* (?= */ -#define META_LOOKAHEADNOT 0x80230000u /* (?! */ -#define META_LOOKBEHIND 0x80240000u /* (?<= */ -#define META_LOOKBEHINDNOT 0x80250000u /* (?flags & PCRE2_DEREF_TABLES) != 0) { /* Decoded tables belong to the codes after deserialization, and they must - be freed when there are no more reference to them. The *ref_count should + be freed when there are no more references to them. The *ref_count should always be > 0. */ ref_count = (PCRE2_SIZE *)(code->tables + tables_length); @@ -1398,7 +1449,7 @@ Arguments: errorcodeptr points to the errorcode variable (containing zero) options the current options bits isclass TRUE if inside a character class - cb compile data block + cb compile data block or NULL when called from pcre2_substitute() Returns: zero => a data character positive => a special escape sequence @@ -1408,7 +1459,8 @@ Returns: zero => a data character int PRIV(check_escape)(PCRE2_SPTR *ptrptr, PCRE2_SPTR ptrend, uint32_t *chptr, - int *errorcodeptr, uint32_t options, BOOL isclass, compile_block *cb) + int *errorcodeptr, uint32_t options, uint32_t extra_options, BOOL isclass, + compile_block *cb) { BOOL utf = (options & PCRE2_UTF) != 0; PCRE2_SPTR ptr = *ptrptr; @@ -1429,14 +1481,25 @@ GETCHARINCTEST(c, ptr); /* Get character value, increment pointer */ /* Non-alphanumerics are literals, so we just leave the value in c. An initial value test saves a memory lookup for code points outside the alphanumeric -range. Otherwise, do a table lookup. A non-zero result is something that can be -returned immediately. Otherwise further processing is required. */ +range. */ if (c < ESCAPES_FIRST || c > ESCAPES_LAST) {} /* Definitely literal */ +/* Otherwise, do a table lookup. Non-zero values need little processing here. A +positive value is a literal value for something like \n. A negative value is +the negation of one of the ESC_ macros that is passed back for handling by the +calling function. Some extra checking is needed for \N because only \N{U+dddd} +is supported. If the value is zero, further processing is handled below. */ + else if ((i = escapes[c - ESCAPES_FIRST]) != 0) { - if (i > 0) c = (uint32_t)i; else /* Positive is a data character */ + if (i > 0) + { + c = (uint32_t)i; + if (c == CHAR_CR && (extra_options & PCRE2_EXTRA_ESCAPED_CR_IS_LF) != 0) + c = CHAR_LF; + } + else /* Negative table entry */ { escape = -i; /* Else return a special escape */ if (cb != NULL && (escape == ESC_P || escape == ESC_p || escape == ESC_X)) @@ -1486,23 +1549,29 @@ else if ((i = escapes[c - ESCAPES_FIRST]) != 0) } } -/* Escapes that need further processing, including those that are unknown. -When called from pcre2_substitute(), only \c, \o, and \x are recognized (and \u -when BSUX is set). */ +/* Escapes that need further processing, including those that are unknown, have +a zero entry in the lookup table. When called from pcre2_substitute(), only \c, +\o, and \x are recognized (\u and \U can never appear as they are used for case +forcing). */ else { + int s; PCRE2_SPTR oldptr; BOOL overflow; - int s; + BOOL alt_bsux = + ((options & PCRE2_ALT_BSUX) | (extra_options & PCRE2_EXTRA_ALT_BSUX)) != 0; /* Filter calls from pcre2_substitute(). */ - if (cb == NULL && c != CHAR_c && c != CHAR_o && c != CHAR_x && - (c != CHAR_u || (options & PCRE2_ALT_BSUX) != 0)) + if (cb == NULL) { - *errorcodeptr = ERR3; - return 0; + if (c != CHAR_c && c != CHAR_o && c != CHAR_x) + { + *errorcodeptr = ERR3; + return 0; + } + alt_bsux = FALSE; /* Do not modify \x handling */ } switch (c) @@ -1516,40 +1585,75 @@ else *errorcodeptr = ERR37; break; - /* \u is unrecognized when PCRE2_ALT_BSUX is not set. When it is treated - specially, \u must be followed by four hex digits. Otherwise it is a - lowercase u letter. */ + /* \u is unrecognized when neither PCRE2_ALT_BSUX nor PCRE2_EXTRA_ALT_BSUX + is set. Otherwise, \u must be followed by exactly four hex digits or, if + PCRE2_EXTRA_ALT_BSUX is set, by any number of hex digits in braces. + Otherwise it is a lowercase u letter. This gives some compatibility with + ECMAScript (aka JavaScript). */ case CHAR_u: - if ((options & PCRE2_ALT_BSUX) == 0) *errorcodeptr = ERR37; else + if (!alt_bsux) *errorcodeptr = ERR37; else { uint32_t xc; - if (ptrend - ptr < 4) break; /* Less than 4 chars */ - if ((cc = XDIGIT(ptr[0])) == 0xff) break; /* Not a hex digit */ - if ((xc = XDIGIT(ptr[1])) == 0xff) break; /* Not a hex digit */ - cc = (cc << 4) | xc; - if ((xc = XDIGIT(ptr[2])) == 0xff) break; /* Not a hex digit */ - cc = (cc << 4) | xc; - if ((xc = XDIGIT(ptr[3])) == 0xff) break; /* Not a hex digit */ - c = (cc << 4) | xc; - ptr += 4; + + if (ptr >= ptrend) break; + if (*ptr == CHAR_LEFT_CURLY_BRACKET && + (extra_options & PCRE2_EXTRA_ALT_BSUX) != 0) + { + PCRE2_SPTR hptr = ptr + 1; + cc = 0; + + while (hptr < ptrend && (xc = XDIGIT(*hptr)) != 0xff) + { + if ((cc & 0xf0000000) != 0) /* Test for 32-bit overflow */ + { + *errorcodeptr = ERR77; + ptr = hptr; /* Show where */ + break; /* *hptr != } will cause another break below */ + } + cc = (cc << 4) | xc; + hptr++; + } + + if (hptr == ptr + 1 || /* No hex digits */ + hptr >= ptrend || /* Hit end of input */ + *hptr != CHAR_RIGHT_CURLY_BRACKET) /* No } terminator */ + break; /* Hex escape not recognized */ + + c = cc; /* Accept the code point */ + ptr = hptr + 1; + } + + else /* Must be exactly 4 hex digits */ + { + if (ptrend - ptr < 4) break; /* Less than 4 chars */ + if ((cc = XDIGIT(ptr[0])) == 0xff) break; /* Not a hex digit */ + if ((xc = XDIGIT(ptr[1])) == 0xff) break; /* Not a hex digit */ + cc = (cc << 4) | xc; + if ((xc = XDIGIT(ptr[2])) == 0xff) break; /* Not a hex digit */ + cc = (cc << 4) | xc; + if ((xc = XDIGIT(ptr[3])) == 0xff) break; /* Not a hex digit */ + c = (cc << 4) | xc; + ptr += 4; + } + if (utf) { if (c > 0x10ffffU) *errorcodeptr = ERR77; else if (c >= 0xd800 && c <= 0xdfff && - (cb->cx->extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0) - *errorcodeptr = ERR73; + (extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0) + *errorcodeptr = ERR73; } else if (c > MAX_NON_UTF_CHAR) *errorcodeptr = ERR77; } break; - /* \U is unrecognized unless PCRE2_ALT_BSUX is set, in which case it is an - upper case letter. */ + /* \U is unrecognized unless PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX is set, + in which case it is an upper case letter. */ case CHAR_U: - if ((options & PCRE2_ALT_BSUX) == 0) *errorcodeptr = ERR37; + if (!alt_bsux) *errorcodeptr = ERR37; break; /* In a character class, \g is just a literal "g". Outside a character @@ -1728,8 +1832,8 @@ else } else if (ptr < ptrend && *ptr++ == CHAR_RIGHT_CURLY_BRACKET) { - if (utf && c >= 0xd800 && c <= 0xdfff && (cb == NULL || - (cb->cx->extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0)) + if (utf && c >= 0xd800 && c <= 0xdfff && + (extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0) { ptr--; *errorcodeptr = ERR73; @@ -1743,11 +1847,11 @@ else } break; - /* \x is complicated. When PCRE2_ALT_BSUX is set, \x must be followed by - two hexadecimal digits. Otherwise it is a lowercase x letter. */ + /* When PCRE2_ALT_BSUX or PCRE2_EXTRA_ALT_BSUX is set, \x must be followed + by two hexadecimal digits. Otherwise it is a lowercase x letter. */ case CHAR_x: - if ((options & PCRE2_ALT_BSUX) != 0) + if (alt_bsux) { uint32_t xc; if (ptrend - ptr < 2) break; /* Less than 2 characters */ @@ -1755,9 +1859,9 @@ else if ((xc = XDIGIT(ptr[1])) == 0xff) break; /* Not a hex digit */ c = (cc << 4) | xc; ptr += 2; - } /* End PCRE2_ALT_BSUX handling */ + } - /* Handle \x in Perl's style. \x{ddd} is a character number which can be + /* Handle \x in Perl's style. \x{ddd} is a character code which can be greater than 0xff in UTF-8 or non-8bit mode, but only if the ddd are hex digits. If not, { used to be treated as a data character. However, Perl seems to read hex digits up to the first non-such, and ignore the rest, so @@ -1801,8 +1905,8 @@ else } else if (ptr < ptrend && *ptr++ == CHAR_RIGHT_CURLY_BRACKET) { - if (utf && c >= 0xd800 && c <= 0xdfff && (cb == NULL || - (cb->cx->extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0)) + if (utf && c >= 0xd800 && c <= 0xdfff && + (extra_options & PCRE2_EXTRA_ALLOW_SURROGATE_ESCAPES) == 0) { ptr--; *errorcodeptr = ERR73; @@ -1874,9 +1978,9 @@ else c ^= 0x40; /* Handle \c in an EBCDIC environment. The special case \c? is converted to - 255 (0xff) or 95 (0x5f) if other character suggest we are using th POSIX-BC - encoding. (This is the way Perl indicates that it handles \c?.) The other - valid sequences correspond to a list of specific characters. */ + 255 (0xff) or 95 (0x5f) if other characters suggest we are using the + POSIX-BC encoding. (This is the way Perl indicates that it handles \c?.) + The other valid sequences correspond to a list of specific characters. */ #else if (c == CHAR_QUESTION_MARK) @@ -2120,9 +2224,10 @@ return -1; *************************************************/ /* This function is called from parse_regex() below whenever it needs to read -the name of a subpattern or a (*VERB). The initial pointer must be to the -character before the name. If that character is '*' we are reading a verb name. -The pointer is updated to point after the name, for a VERB, or after tha name's +the name of a subpattern or a (*VERB) or an (*alpha_assertion). The initial +pointer must be to the character before the name. If that character is '*' we +are reading a verb or alpha assertion name. The pointer is updated to point +after the name, for a VERB or alpha assertion name, or after tha name's terminator for a subpattern name. Returning both the offset and the name pointer is redundant information, but some callers use one and some the other, so it is simplest just to return both. @@ -2130,6 +2235,7 @@ so it is simplest just to return both. Arguments: ptrptr points to the character pointer variable ptrend points to the end of the input string + utf true if the input is UTF-encoded terminator the terminator of a subpattern name must be this offsetptr where to put the offset from the start of the pattern nameptr where to put a pointer to the name in the input @@ -2142,48 +2248,88 @@ Returns: TRUE if a name was read */ static BOOL -read_name(PCRE2_SPTR *ptrptr, PCRE2_SPTR ptrend, uint32_t terminator, +read_name(PCRE2_SPTR *ptrptr, PCRE2_SPTR ptrend, BOOL utf, uint32_t terminator, PCRE2_SIZE *offsetptr, PCRE2_SPTR *nameptr, uint32_t *namelenptr, int *errorcodeptr, compile_block *cb) { PCRE2_SPTR ptr = *ptrptr; -BOOL is_verb = (*ptr == CHAR_ASTERISK); -uint32_t namelen = 0; -uint32_t ctype = is_verb? ctype_letter : ctype_word; +BOOL is_group = (*ptr != CHAR_ASTERISK); -if (++ptr >= ptrend) +if (++ptr >= ptrend) /* No characters in name */ { - *errorcodeptr = is_verb? ERR60: /* Verb not recognized or malformed */ - ERR62; /* Subpattern name expected */ + *errorcodeptr = is_group? ERR62: /* Subpattern name expected */ + ERR60; /* Verb not recognized or malformed */ goto FAILED; } *nameptr = ptr; *offsetptr = (PCRE2_SIZE)(ptr - cb->start_pattern); -if (IS_DIGIT(*ptr)) - { - *errorcodeptr = ERR44; /* Group name must not start with digit */ - goto FAILED; - } +/* In UTF mode, a group name may contain letters and decimal digits as defined +by Unicode properties, and underscores, but must not start with a digit. */ -while (ptr < ptrend && MAX_255(*ptr) && (cb->ctypes[*ptr] & ctype) != 0) +#ifdef SUPPORT_UNICODE +if (utf && is_group) { - ptr++; - namelen++; - if (namelen > MAX_NAME_SIZE) + uint32_t c, type; + + GETCHAR(c, ptr); + type = UCD_CHARTYPE(c); + + if (type == ucp_Nd) { - *errorcodeptr = ERR48; + *errorcodeptr = ERR44; goto FAILED; } + + for(;;) + { + if (type != ucp_Nd && PRIV(ucp_gentype)[type] != ucp_L && + c != CHAR_UNDERSCORE) break; + ptr++; + FORWARDCHARTEST(ptr, ptrend); + if (ptr >= ptrend) break; + GETCHAR(c, ptr); + type = UCD_CHARTYPE(c); + } + } +else +#else +(void)utf; /* Avoid compiler warning */ +#endif /* SUPPORT_UNICODE */ + +/* Handle non-group names and group names in non-UTF modes. A group name must +not start with a digit. If either of the others start with a digit it just +won't be recognized. */ + + { + if (is_group && IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR44; + goto FAILED; + } + + while (ptr < ptrend && MAX_255(*ptr) && (cb->ctypes[*ptr] & ctype_word) != 0) + { + ptr++; + } } -/* Subpattern names must not be empty, and their terminator is checked here. -(What follows a verb name is checked separately.) */ +/* Check name length */ -if (!is_verb) +if (ptr > *nameptr + MAX_NAME_SIZE) { - if (namelen == 0) + *errorcodeptr = ERR48; + goto FAILED; + } +*namelenptr = ptr - *nameptr; + +/* Subpattern names must not be empty, and their terminator is checked here. +(What follows a verb or alpha assertion name is checked separately.) */ + +if (is_group) + { + if (ptr == *nameptr) { *errorcodeptr = ERR62; /* Subpattern name expected */ goto FAILED; @@ -2196,7 +2342,6 @@ if (!is_verb) ptr++; } -*namelenptr = namelen; *ptrptr = ptr; return TRUE; @@ -2289,6 +2434,7 @@ typedef struct nest_save { #define NSF_RESET 0x0001u #define NSF_CONDASSERT 0x0002u +#define NSF_ATOMICSR 0x0004u /* Options that are changeable within the pattern must be tracked during parsing. Some (e.g. PCRE2_EXTENDED) are implemented entirely during parsing, @@ -2333,6 +2479,7 @@ uint32_t *parsed_pattern = cb->parsed_pattern; uint32_t *parsed_pattern_end = cb->parsed_pattern_end; uint32_t meta_quantifier = 0; uint32_t add_after_mark = 0; +uint32_t extra_options = cb->cx->extra_options; uint16_t nest_depth = 0; int after_manual_callout = 0; int expect_cond_assert = 0; @@ -2356,12 +2503,12 @@ nest_save *top_nest, *end_nests; /* Insert leading items for word and line matching (features provided for the benefit of pcre2grep). */ -if ((cb->cx->extra_options & PCRE2_EXTRA_MATCH_LINE) != 0) +if ((extra_options & PCRE2_EXTRA_MATCH_LINE) != 0) { *parsed_pattern++ = META_CIRCUMFLEX; *parsed_pattern++ = META_NOCAPTURE; } -else if ((cb->cx->extra_options & PCRE2_EXTRA_MATCH_WORD) != 0) +else if ((extra_options & PCRE2_EXTRA_MATCH_WORD) != 0) { *parsed_pattern++ = META_ESCAPE + ESC_b; *parsed_pattern++ = META_NOCAPTURE; @@ -2526,7 +2673,7 @@ while (ptr < ptrend) if ((options & PCRE2_ALT_VERBNAMES) != 0) { escape = PRIV(check_escape)(&ptr, ptrend, &c, &errorcode, options, - FALSE, cb); + cb->cx->extra_options, FALSE, cb); if (errorcode != 0) goto FAILED; } else escape = 0; /* Treat all as literal */ @@ -2639,23 +2786,30 @@ while (ptr < ptrend) if (expect_cond_assert > 0) { BOOL ok = c == CHAR_LEFT_PARENTHESIS && ptrend - ptr >= 3 && - ptr[0] == CHAR_QUESTION_MARK; - if (ok) switch(ptr[1]) + (ptr[0] == CHAR_QUESTION_MARK || ptr[0] == CHAR_ASTERISK); + if (ok) { - case CHAR_C: - ok = expect_cond_assert == 2; - break; + if (ptr[0] == CHAR_ASTERISK) /* New alpha assertion format, possibly */ + { + ok = MAX_255(ptr[1]) && (cb->ctypes[ptr[1]] & ctype_lcletter) != 0; + } + else switch(ptr[1]) /* Traditional symbolic format */ + { + case CHAR_C: + ok = expect_cond_assert == 2; + break; - case CHAR_EQUALS_SIGN: - case CHAR_EXCLAMATION_MARK: - break; + case CHAR_EQUALS_SIGN: + case CHAR_EXCLAMATION_MARK: + break; - case CHAR_LESS_THAN_SIGN: - ok = ptr[2] == CHAR_EQUALS_SIGN || ptr[2] == CHAR_EXCLAMATION_MARK; - break; + case CHAR_LESS_THAN_SIGN: + ok = ptr[2] == CHAR_EQUALS_SIGN || ptr[2] == CHAR_EXCLAMATION_MARK; + break; - default: - ok = FALSE; + default: + ok = FALSE; + } } if (!ok) @@ -2709,11 +2863,11 @@ while (ptr < ptrend) case CHAR_BACKSLASH: tempptr = ptr; escape = PRIV(check_escape)(&ptr, ptrend, &c, &errorcode, options, - FALSE, cb); + cb->cx->extra_options, FALSE, cb); if (errorcode != 0) { ESCAPE_FAILED: - if ((cb->cx->extra_options & PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL) == 0) + if ((extra_options & PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL) == 0) goto FAILED; ptr = tempptr; if (ptr >= ptrend) c = CHAR_BACKSLASH; else @@ -2907,7 +3061,7 @@ while (ptr < ptrend) /* Not a numerical recursion */ - if (!read_name(&ptr, ptrend, terminator, &offset, &name, &namelen, + if (!read_name(&ptr, ptrend, utf, terminator, &offset, &name, &namelen, &errorcode, cb)) goto ESCAPE_FAILED; /* \k and \g when used with braces are back references, whereas \g used @@ -3270,12 +3424,12 @@ while (ptr < ptrend) else { tempptr = ptr; - escape = PRIV(check_escape)(&ptr, ptrend, &c, &errorcode, - options, TRUE, cb); + escape = PRIV(check_escape)(&ptr, ptrend, &c, &errorcode, options, + cb->cx->extra_options, TRUE, cb); + if (errorcode != 0) { - CLASS_ESCAPE_FAILED: - if ((cb->cx->extra_options & PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL) == 0) + if ((extra_options & PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL) == 0) goto FAILED; ptr = tempptr; if (ptr >= ptrend) c = CHAR_BACKSLASH; else @@ -3285,30 +3439,32 @@ while (ptr < ptrend) escape = 0; /* Treat as literal character */ } - if (escape == 0) /* Escaped character code point is in c */ + switch(escape) { + case 0: /* Escaped character code point is in c */ char_is_literal = FALSE; goto CLASS_LITERAL; - } - /* These three escapes do not alter the class range state. */ - - if (escape == ESC_b) - { - c = CHAR_BS; /* \b is backspace in a class */ + case ESC_b: + c = CHAR_BS; /* \b is backspace in a class */ char_is_literal = FALSE; goto CLASS_LITERAL; - } - else if (escape == ESC_Q) - { + case ESC_Q: inescq = TRUE; /* Enter literal mode */ goto CLASS_CONTINUE; - } - else if (escape == ESC_E) /* Ignore orphan \E */ + case ESC_E: /* Ignore orphan \E */ goto CLASS_CONTINUE; + case ESC_B: /* Always an error in a class */ + case ESC_R: + case ESC_X: + errorcode = ERR7; + ptr--; + goto FAILED; + } + /* The second part of a range can be a single-character escape sequence (detected above), but not any of the other escapes. Perl treats a hyphen as a literal in such circumstances. However, in Perl's @@ -3318,7 +3474,7 @@ while (ptr < ptrend) if (class_range_state == RANGE_STARTED) { errorcode = ERR50; - goto CLASS_ESCAPE_FAILED; + goto FAILED; /* Not CLASS_ESCAPE_FAILED; always an error */ } /* Of the remaining escapes, only those that define characters are @@ -3328,8 +3484,8 @@ while (ptr < ptrend) switch(escape) { case ESC_N: - errorcode = ERR71; /* Not supported in a class */ - goto CLASS_ESCAPE_FAILED; + errorcode = ERR71; + goto FAILED; case ESC_H: case ESC_h: @@ -3392,14 +3548,14 @@ while (ptr < ptrend) } #else errorcode = ERR45; - goto CLASS_ESCAPE_FAILED; + goto FAILED; #endif break; /* End \P and \p */ default: /* All others are not allowed in a class */ errorcode = ERR7; ptr--; - goto CLASS_ESCAPE_FAILED; + goto FAILED; } /* Perl gives a warning unless a following hyphen is the last character @@ -3440,7 +3596,8 @@ while (ptr < ptrend) case CHAR_LEFT_PARENTHESIS: if (ptr >= ptrend) goto UNCLOSED_PARENTHESIS; - /* If ( is not followed by ? it is either a capture or a special verb. */ + /* If ( is not followed by ? it is either a capture or a special verb or an + alpha assertion. */ if (*ptr != CHAR_QUESTION_MARK) { @@ -3460,17 +3617,122 @@ while (ptr < ptrend) else *parsed_pattern++ = META_NOCAPTURE; } + /* Do nothing for (* followed by end of pattern or ) so it gives a "bad + quantifier" error rather than "(*MARK) must have an argument". */ + + else if (ptrend - ptr <= 1 || (c = ptr[1]) == CHAR_RIGHT_PARENTHESIS) + break; + + /* Handle "alpha assertions" such as (*pla:...). Most of these are + synonyms for the historical symbolic assertions, but the script run ones + are new. They are distinguished by starting with a lower case letter. + Checking both ends of the alphabet makes this work in all character + codes. */ + + else if (CHMAX_255(c) && (cb->ctypes[c] & ctype_lcletter) != 0) + { + uint32_t meta; + + vn = alasnames; + if (!read_name(&ptr, ptrend, utf, 0, &offset, &name, &namelen, + &errorcode, cb)) goto FAILED; + if (ptr >= ptrend || *ptr != CHAR_COLON) + { + errorcode = ERR95; /* Malformed */ + goto FAILED; + } + + /* Scan the table of alpha assertion names */ + + for (i = 0; i < alascount; i++) + { + if (namelen == alasmeta[i].len && + PRIV(strncmp_c8)(name, vn, namelen) == 0) + break; + vn += alasmeta[i].len + 1; + } + + if (i >= alascount) + { + errorcode = ERR95; /* Alpha assertion not recognized */ + goto FAILED; + } + + /* Check for expecting an assertion condition. If so, only lookaround + assertions are valid. */ + + meta = alasmeta[i].meta; + if (prev_expect_cond_assert > 0 && + (meta < META_LOOKAHEAD || meta > META_LOOKBEHINDNOT)) + { + errorcode = ERR28; /* Assertion expected */ + goto FAILED; + } + + /* The lookaround alphabetic synonyms can be almost entirely handled by + jumping to the code that handles the traditional symbolic forms. */ + + switch(meta) + { + default: + errorcode = ERR89; /* Unknown code; should never occur because */ + goto FAILED; /* the meta values come from a table above. */ + + case META_ATOMIC: + goto ATOMIC_GROUP; + + case META_LOOKAHEAD: + goto POSITIVE_LOOK_AHEAD; + + case META_LOOKAHEADNOT: + goto NEGATIVE_LOOK_AHEAD; + + case META_LOOKBEHIND: + case META_LOOKBEHINDNOT: + *parsed_pattern++ = meta; + ptr--; + goto POST_LOOKBEHIND; + + /* The script run facilities are handled here. Unicode support is + required (give an error if not, as this is a security issue). Always + record a META_SCRIPT_RUN item. Then, for the atomic version, insert + META_ATOMIC and remember that we need two META_KETs at the end. */ + + case META_SCRIPT_RUN: + case META_ATOMIC_SCRIPT_RUN: +#ifdef SUPPORT_UNICODE + *parsed_pattern++ = META_SCRIPT_RUN; + nest_depth++; + ptr++; + if (meta == META_ATOMIC_SCRIPT_RUN) + { + *parsed_pattern++ = META_ATOMIC; + if (top_nest == NULL) top_nest = (nest_save *)(cb->start_workspace); + else if (++top_nest >= end_nests) + { + errorcode = ERR84; + goto FAILED; + } + top_nest->nest_depth = nest_depth; + top_nest->flags = NSF_ATOMICSR; + top_nest->options = options & PARSE_TRACKED_OPTIONS; + } + break; +#else /* SUPPORT_UNICODE */ + errorcode = ERR96; + goto FAILED; +#endif + } + } + /* ---- Handle (*VERB) and (*VERB:NAME) ---- */ - /* Do nothing for (*) so it gives a "bad quantifier" error rather than - "(*MARK) must have an argument". */ - - else if (ptrend - ptr > 1 && ptr[1] != CHAR_RIGHT_PARENTHESIS) + else { vn = verbnames; - if (!read_name(&ptr, ptrend, 0, &offset, &name, &namelen, &errorcode, - cb)) goto FAILED; + if (!read_name(&ptr, ptrend, utf, 0, &offset, &name, &namelen, + &errorcode, cb)) goto FAILED; if (ptr >= ptrend || (*ptr != CHAR_COLON && *ptr != CHAR_RIGHT_PARENTHESIS)) { @@ -3725,7 +3987,7 @@ while (ptr < ptrend) errorcode = ERR41; goto FAILED; } - if (!read_name(&ptr, ptrend, CHAR_RIGHT_PARENTHESIS, &offset, &name, + if (!read_name(&ptr, ptrend, utf, CHAR_RIGHT_PARENTHESIS, &offset, &name, &namelen, &errorcode, cb)) goto FAILED; *parsed_pattern++ = META_BACKREF_BYNAME; *parsed_pattern++ = namelen; @@ -3785,7 +4047,7 @@ while (ptr < ptrend) case CHAR_AMPERSAND: RECURSE_BY_NAME: - if (!read_name(&ptr, ptrend, CHAR_RIGHT_PARENTHESIS, &offset, &name, + if (!read_name(&ptr, ptrend, utf, CHAR_RIGHT_PARENTHESIS, &offset, &name, &namelen, &errorcode, cb)) goto FAILED; *parsed_pattern++ = META_RECURSE_BYNAME; *parsed_pattern++ = namelen; @@ -3933,14 +4195,15 @@ while (ptr < ptrend) if (++ptr >= ptrend) goto UNCLOSED_PARENTHESIS; nest_depth++; - /* If the next character is ? there must be an assertion next (optionally - preceded by a callout). We do not check this here, but instead we set - expect_cond_assert to 2. If this is still greater than zero (callouts - decrement it) when the next assertion is read, it will be marked as a - condition that must not be repeated. A value greater than zero also - causes checking that an assertion (possibly with callout) follows. */ + /* If the next character is ? or * there must be an assertion next + (optionally preceded by a callout). We do not check this here, but + instead we set expect_cond_assert to 2. If this is still greater than + zero (callouts decrement it) when the next assertion is read, it will be + marked as a condition that must not be repeated. A value greater than + zero also causes checking that an assertion (possibly with callout) + follows. */ - if (*ptr == CHAR_QUESTION_MARK) + if (*ptr == CHAR_QUESTION_MARK || *ptr == CHAR_ASTERISK) { *parsed_pattern++ = META_COND_ASSERT; ptr--; /* Pull pointer back to the opening parenthesis. */ @@ -4032,7 +4295,7 @@ while (ptr < ptrend) terminator = CHAR_RIGHT_PARENTHESIS; ptr--; /* Point to char before name */ } - if (!read_name(&ptr, ptrend, terminator, &offset, &name, &namelen, + if (!read_name(&ptr, ptrend, utf, terminator, &offset, &name, &namelen, &errorcode, cb)) goto FAILED; /* Handle (?(R&name) */ @@ -4086,6 +4349,7 @@ while (ptr < ptrend) /* ---- Atomic group ---- */ case CHAR_GREATER_THAN_SIGN: + ATOMIC_GROUP: /* Come from (*atomic: */ *parsed_pattern++ = META_ATOMIC; nest_depth++; ptr++; @@ -4095,11 +4359,13 @@ while (ptr < ptrend) /* ---- Lookahead assertions ---- */ case CHAR_EQUALS_SIGN: + POSITIVE_LOOK_AHEAD: /* Come from (*pla: */ *parsed_pattern++ = META_LOOKAHEAD; ptr++; goto POST_ASSERTION; case CHAR_EXCLAMATION_MARK: + NEGATIVE_LOOK_AHEAD: /* Come from (*nla: */ *parsed_pattern++ = META_LOOKAHEADNOT; ptr++; goto POST_ASSERTION; @@ -4119,6 +4385,8 @@ while (ptr < ptrend) } *parsed_pattern++ = (ptr[1] == CHAR_EQUALS_SIGN)? META_LOOKBEHIND : META_LOOKBEHINDNOT; + + POST_LOOKBEHIND: /* Come from (*plb: and (*nlb: */ *has_lookbehind = TRUE; offset = (PCRE2_SIZE)(ptr - cb->start_pattern - 2); PUTOFFSET(offset, parsed_pattern); @@ -4161,7 +4429,7 @@ while (ptr < ptrend) terminator = CHAR_APOSTROPHE; /* Terminator */ DEFINE_NAME: - if (!read_name(&ptr, ptrend, terminator, &offset, &name, &namelen, + if (!read_name(&ptr, ptrend, utf, terminator, &offset, &name, &namelen, &errorcode, cb)) goto FAILED; /* We have a name for this capturing group. It is also assigned a number, @@ -4280,6 +4548,14 @@ while (ptr < ptrend) cb->bracount = top_nest->max_group; if ((top_nest->flags & NSF_CONDASSERT) != 0) okquantifier = FALSE; + + if ((top_nest->flags & NSF_ATOMICSR) != 0) + { + *parsed_pattern++ = META_KET; + } + + + if (top_nest == (nest_save *)(cb->start_workspace)) top_nest = NULL; else top_nest--; } @@ -4311,12 +4587,12 @@ parsed_pattern = manage_callouts(ptr, &previous_callout, auto_callout, /* Insert trailing items for word and line matching (features provided for the benefit of pcre2grep). */ -if ((cb->cx->extra_options & PCRE2_EXTRA_MATCH_LINE) != 0) +if ((extra_options & PCRE2_EXTRA_MATCH_LINE) != 0) { *parsed_pattern++ = META_KET; *parsed_pattern++ = META_DOLLAR; } -else if ((cb->cx->extra_options & PCRE2_EXTRA_MATCH_WORD) != 0) +else if ((extra_options & PCRE2_EXTRA_MATCH_WORD) != 0) { *parsed_pattern++ = META_KET; *parsed_pattern++ = META_ESCAPE + ESC_b; @@ -4421,6 +4697,14 @@ for (;;) code += GET(code, 1) + 1 + LINK_SIZE; break; + case OP_MARK: + case OP_COMMIT_ARG: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1] + PRIV(OP_lengths)[*code]; + break; + default: return code; } @@ -5516,10 +5800,10 @@ for (;; pptr++) if (range_is_literal && (cb->ctypes[c] & ctype_letter) != 0 && (cb->ctypes[d] & ctype_letter) != 0 && - (d <= CHAR_z) == (d <= CHAR_z)) + (c <= CHAR_z) == (d <= CHAR_z)) { uint32_t uc = (d <= CHAR_z)? 0 : 64; - uint32_t C = d - uc; + uint32_t C = c - uc; uint32_t D = d - uc; if (C <= CHAR_i) @@ -5664,7 +5948,10 @@ for (;; pptr++) (void)memmove(code + (32 / sizeof(PCRE2_UCHAR)), code, CU2BYTES(class_uchardata - code)); if (negate_class && !xclass_has_prop) - for (i = 0; i < 32; i++) classbits[i] = ~classbits[i]; + { + /* Using 255 ^ instead of ~ avoids clang sanitize warning. */ + for (i = 0; i < 32; i++) classbits[i] = 255 ^ classbits[i]; + } memcpy(code, classbits, 32); code = class_uchardata + (32 / sizeof(PCRE2_UCHAR)); } @@ -5687,7 +5974,10 @@ for (;; pptr++) if (lengthptr == NULL) /* Save time in the pre-compile phase */ { if (negate_class) - for (i = 0; i < 32; i++) classbits[i] = ~classbits[i]; + { + /* Using 255 ^ instead of ~ avoids clang sanitize warning. */ + for (i = 0; i < 32; i++) classbits[i] = 255 ^ classbits[i]; + } memcpy(code, classbits, 32); } code += 32 / sizeof(PCRE2_UCHAR); @@ -5901,7 +6191,7 @@ for (;; pptr++) } goto GROUP_PROCESS_NOTE_EMPTY; - /* The DEFINE condition is always false. It's internal groups may never + /* The DEFINE condition is always false. Its internal groups may never be called, so matched_char must remain false, hence the jump to GROUP_PROCESS rather than GROUP_PROCESS_NOTE_EMPTY. */ @@ -5997,6 +6287,10 @@ for (;; pptr++) bravalue = OP_ONCE; goto GROUP_PROCESS_NOTE_EMPTY; + case META_SCRIPT_RUN: + bravalue = OP_SCRIPT_RUN; + goto GROUP_PROCESS_NOTE_EMPTY; + case META_NOCAPTURE: bravalue = OP_BRA; /* Fall through */ @@ -6237,8 +6531,8 @@ for (;; pptr++) groupnumber = ng->number; /* For a recursion, that's all that is needed. We can now go to - the code above that handles numerical recursion, applying it to - the first group with the given name. */ + the code that handles numerical recursion, applying it to the first + group with the given name. */ if (meta == META_RECURSE_BYNAME) { @@ -6632,6 +6926,7 @@ for (;; pptr++) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRA: case OP_CBRA: case OP_COND: @@ -6844,16 +7139,16 @@ for (;; pptr++) } /* If the maximum is unlimited, set a repeater in the final copy. For - ONCE brackets, that's all we need to do. However, possessively repeated - ONCE brackets can be converted into non-capturing brackets, as the - behaviour of (?:xx)++ is the same as (?>xx)++ and this saves having to - deal with possessive ONCEs specially. + SCRIPT_RUN and ONCE brackets, that's all we need to do. However, + possessively repeated ONCE brackets can be converted into non-capturing + brackets, as the behaviour of (?:xx)++ is the same as (?>xx)++ and this + saves having to deal with possessive ONCEs specially. Otherwise, when we are doing the actual compile phase, check to see whether this group is one that could match an empty string. If so, convert the initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so that runtime checking can be done. [This check is also applied to ONCE - groups at runtime, but in a different way.] + and SCRIPT_RUN groups at runtime, but in a different way.] Then, if the quantifier was possessive and the bracket is not a conditional, we convert the BRA code to the POS form, and the KET code to @@ -6877,13 +7172,14 @@ for (;; pptr++) if (*bracode == OP_ONCE && possessive_quantifier) *bracode = OP_BRA; - /* For non-possessive ONCE brackets, all we need to do is to - set the KET. */ + /* For non-possessive ONCE and for SCRIPT_RUN brackets, all we need + to do is to set the KET. */ - if (*bracode == OP_ONCE) *ketcode = OP_KETRMAX + repeat_type; + if (*bracode == OP_ONCE || *bracode == OP_SCRIPT_RUN) + *ketcode = OP_KETRMAX + repeat_type; - /* Handle non-ONCE brackets and possessive ONCEs (which have been - converted to non-capturing above). */ + /* Handle non-SCRIPT_RUN and non-ONCE brackets and possessive ONCEs + (which have been converted to non-capturing above). */ else { @@ -7267,9 +7563,8 @@ for (;; pptr++) scanned and these numbers are replaced by offsets within the pattern. It is done like this to avoid problems with forward references and adjusting offsets when groups are duplicated and moved (as discovered in previous - implementations). Note that a recursion does not have a set first character - (relevant if it is repeated, because it will then be wrapped with ONCE - brackets). */ + implementations). Note that a recursion does not have a set first + character. */ case META_RECURSE: GETPLUSOFFSET(offset, pptr); @@ -7286,6 +7581,8 @@ for (;; pptr++) groupsetfirstcu = FALSE; cb->had_recurse = TRUE; if (firstcuflags == REQ_UNSET) firstcuflags = REQ_NONE; + zerofirstcu = firstcu; + zerofirstcuflags = firstcuflags; break; @@ -7340,9 +7637,20 @@ for (;; pptr++) { uint32_t ptype = *(++pptr) >> 16; uint32_t pdata = *pptr & 0xffff; - *code++ = (meta_arg == ESC_p)? OP_PROP : OP_NOTPROP; - *code++ = ptype; - *code++ = pdata; + + /* The special case of \p{Any} is compiled to OP_ALLANY so as to benefit + from the auto-anchoring code. */ + + if (meta_arg == ESC_p && ptype == PT_ANY) + { + *code++ = OP_ALLANY; + } + else + { + *code++ = (meta_arg == ESC_p)? OP_PROP : OP_NOTPROP; + *code++ = ptype; + *code++ = pdata; + } break; /* End META_ESCAPE */ } #endif @@ -8240,6 +8548,7 @@ do { case OP_SCBRAPOS: case OP_ASSERT: case OP_ONCE: + case OP_SCRIPT_RUN: d = find_firstassertedcu(scode, &dflags, inassert + ((op==OP_ASSERT)?1:0)); if (dflags < 0) return 0; @@ -8439,6 +8748,7 @@ for (;; pptr++) case META_LOOKBEHIND: case META_LOOKBEHINDNOT: case META_NOCAPTURE: + case META_SCRIPT_RUN: nestlevel++; break; @@ -8851,6 +9161,7 @@ for (;; pptr++) case META_ATOMIC: case META_NOCAPTURE: + case META_SCRIPT_RUN: pptr++; CHECK_GROUP: grouplength = get_grouplength(&pptr, TRUE, errcodeptr, lcptr, group, @@ -9030,6 +9341,7 @@ for (pptr = cb->parsed_pattern; *pptr != META_END; pptr++) case META_QUERY_QUERY: case META_RANGE_ESCAPED: case META_RANGE_LITERAL: + case META_SCRIPT_RUN: case META_SKIP: case META_THEN: break; diff --git a/thirdparty/pcre2/src/pcre2_context.c b/thirdparty/pcre2/src/pcre2_context.c index 2c14df00805..9c2886a6d02 100644 --- a/thirdparty/pcre2/src/pcre2_context.c +++ b/thirdparty/pcre2/src/pcre2_context.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2017 University of Cambridge + New API code Copyright (c) 2016-2018 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -163,11 +163,13 @@ when no context is supplied to a match function. */ const pcre2_match_context PRIV(default_match_context) = { { default_malloc, default_free, NULL }, #ifdef SUPPORT_JIT - NULL, - NULL, + NULL, /* JIT callback */ + NULL, /* JIT callback data */ #endif - NULL, - NULL, + NULL, /* Callout function */ + NULL, /* Callout data */ + NULL, /* Substitute callout function */ + NULL, /* Substitute callout data */ PCRE2_UNSET, /* Offset limit */ HEAP_LIMIT, MATCH_LIMIT, @@ -403,6 +405,16 @@ mcontext->callout_data = callout_data; return 0; } +PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION +pcre2_set_substitute_callout(pcre2_match_context *mcontext, + int (*substitute_callout)(pcre2_substitute_callout_block *, void *), + void *substitute_callout_data) +{ +mcontext->substitute_callout = substitute_callout; +mcontext->substitute_callout_data = substitute_callout_data; +return 0; +} + PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION pcre2_set_heap_limit(pcre2_match_context *mcontext, uint32_t limit) { diff --git a/thirdparty/pcre2/src/pcre2_convert.c b/thirdparty/pcre2/src/pcre2_convert.c index 1dd5c337dc3..d45b6fee974 100644 --- a/thirdparty/pcre2/src/pcre2_convert.c +++ b/thirdparty/pcre2/src/pcre2_convert.c @@ -276,7 +276,7 @@ while (plength > 0) break; case CHAR_BACKSLASH: - if (plength <= 0) return PCRE2_ERROR_END_BACKSLASH; + if (plength == 0) return PCRE2_ERROR_END_BACKSLASH; if (extended) nextisliteral = TRUE; else { if (*posix < 127 && strchr(posix_meta_escapes, *posix) != NULL) diff --git a/thirdparty/pcre2/src/pcre2_dfa_match.c b/thirdparty/pcre2/src/pcre2_dfa_match.c index 9b43237da75..bbf3e21064b 100644 --- a/thirdparty/pcre2/src/pcre2_dfa_match.c +++ b/thirdparty/pcre2/src/pcre2_dfa_match.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -85,7 +85,8 @@ in others, so I abandoned this code. */ #define PUBLIC_DFA_MATCH_OPTIONS \ (PCRE2_ANCHORED|PCRE2_ENDANCHORED|PCRE2_NOTBOL|PCRE2_NOTEOL|PCRE2_NOTEMPTY| \ PCRE2_NOTEMPTY_ATSTART|PCRE2_NO_UTF_CHECK|PCRE2_PARTIAL_HARD| \ - PCRE2_PARTIAL_SOFT|PCRE2_DFA_SHORTEST|PCRE2_DFA_RESTART) + PCRE2_PARTIAL_SOFT|PCRE2_DFA_SHORTEST|PCRE2_DFA_RESTART| \ + PCRE2_COPY_MATCHED_SUBJECT) /************************************************* @@ -173,6 +174,7 @@ static const uint8_t coptable[] = { 0, /* Assert behind */ 0, /* Assert behind not */ 0, /* ONCE */ + 0, /* SCRIPT_RUN */ 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ 0, 0, /* CREF, DNCREF */ @@ -247,6 +249,7 @@ static const uint8_t poptable[] = { 0, /* Assert behind */ 0, /* Assert behind not */ 0, /* ONCE */ + 0, /* SCRIPT_RUN */ 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ 0, 0, /* CREF, DNCREF */ @@ -316,8 +319,8 @@ finding the minimum heap requirement for a match. */ typedef struct RWS_anchor { struct RWS_anchor *next; - unsigned int size; /* Number of ints */ - unsigned int free; /* Number of ints */ + uint32_t size; /* Number of ints */ + uint32_t free; /* Number of ints */ } RWS_anchor; #define RWS_ANCHOR_SIZE (sizeof(RWS_anchor)/sizeof(int)) @@ -413,20 +416,24 @@ if (rws->next != NULL) new = rws->next; } -/* All sizes are in units of sizeof(int), except for mb->heaplimit, which is in -kibibytes. */ +/* Sizes in the RWS_anchor blocks are in units of sizeof(int), but +mb->heap_limit and mb->heap_used are in kibibytes. Play carefully, to avoid +overflow. */ else { - unsigned int newsize = rws->size * 2; - unsigned int heapleft = (unsigned int) - (((1024/sizeof(int))*mb->heap_limit - mb->heap_used)); - if (newsize > heapleft) newsize = heapleft; + uint32_t newsize = (rws->size >= UINT32_MAX/2)? UINT32_MAX/2 : rws->size * 2; + uint32_t newsizeK = newsize/(1024/sizeof(int)); + + if (newsizeK + mb->heap_used > mb->heap_limit) + newsizeK = (uint32_t)(mb->heap_limit - mb->heap_used); + newsize = newsizeK*(1024/sizeof(int)); + if (newsize < RWS_RSIZE + ovecsize + RWS_ANCHOR_SIZE) return PCRE2_ERROR_HEAPLIMIT; new = mb->memctl.malloc(newsize*sizeof(int), mb->memctl.memory_data); if (new == NULL) return PCRE2_ERROR_NOMEMORY; - mb->heap_used += newsize; + mb->heap_used += newsizeK; new->next = NULL; new->size = newsize; rws->next = new; @@ -2560,7 +2567,7 @@ for (;;) if (clen > 0) { isinclass = (c > 255)? (codevalue == OP_NCLASS) : - ((((uint8_t *)(code + 1))[c/8] & (1 << (c&7))) != 0); + ((((uint8_t *)(code + 1))[c/8] & (1u << (c&7))) != 0); } } @@ -2753,7 +2760,7 @@ for (;;) /* There is also an always-true condition */ else if (condcode == OP_TRUE) - { ADD_ACTIVE(state_offset + LINK_SIZE + 2 + IMM2_SIZE, 0); } + { ADD_ACTIVE(state_offset + LINK_SIZE + 2, 0); } /* The only supported version of OP_RREF is for the value RREF_ANY, which means "test if in any recursion". We can't test for specifically @@ -3226,6 +3233,8 @@ pcre2_dfa_match(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, pcre2_match_context *mcontext, int *workspace, PCRE2_SIZE wscount) { int rc; +int was_zero_terminated = 0; + const pcre2_real_code *re = (const pcre2_real_code *)code; PCRE2_SPTR start_match; @@ -3265,7 +3274,11 @@ rws->free = RWS_BASE_SIZE - RWS_ANCHOR_SIZE; /* A length equal to PCRE2_ZERO_TERMINATED implies a zero-terminated subject string. */ -if (length == PCRE2_ZERO_TERMINATED) length = PRIV(strlen)(subject); +if (length == PCRE2_ZERO_TERMINATED) + { + length = PRIV(strlen)(subject); + was_zero_terminated = 1; + } /* Plausibility checks */ @@ -3518,10 +3531,20 @@ if ((re->flags & PCRE2_LASTSET) != 0) } } +/* If the match data block was previously used with PCRE2_COPY_MATCHED_SUBJECT, +free the memory that was obtained. */ + +if ((match_data->flags & PCRE2_MD_COPIED_SUBJECT) != 0) + { + match_data->memctl.free((void *)match_data->subject, + match_data->memctl.memory_data); + match_data->flags &= ~PCRE2_MD_COPIED_SUBJECT; + } + /* Fill in fields that are always returned in the match data. */ match_data->code = re; -match_data->subject = subject; +match_data->subject = NULL; /* Default for no match */ match_data->mark = NULL; match_data->matchedby = PCRE2_MATCHEDBY_DFA_INTERPRETER; @@ -3586,7 +3609,7 @@ for (;;) #if PCRE2_CODE_UNIT_WIDTH != 8 if (c > 255) c = 255; #endif - ok = (start_bits[c/8] & (1 << (c&7))) != 0; + ok = (start_bits[c/8] & (1u << (c&7))) != 0; } } if (!ok) break; @@ -3697,7 +3720,7 @@ for (;;) #if PCRE2_CODE_UNIT_WIDTH != 8 if (c > 255) c = 255; #endif - if ((start_bits[c/8] & (1 << (c&7))) != 0) break; + if ((start_bits[c/8] & (1u << (c&7))) != 0) break; start_match++; } @@ -3816,6 +3839,20 @@ for (;;) match_data->rightchar = (PCRE2_SIZE)( mb->last_used_ptr - subject); match_data->startchar = (PCRE2_SIZE)(start_match - subject); match_data->rc = rc; + + if (rc >= 0 &&(options & PCRE2_COPY_MATCHED_SUBJECT) != 0) + { + length = CU2BYTES(length + was_zero_terminated); + match_data->subject = match_data->memctl.malloc(length, + match_data->memctl.memory_data); + if (match_data->subject == NULL) return PCRE2_ERROR_NOMEMORY; + memcpy((void *)match_data->subject, subject, length); + match_data->flags |= PCRE2_MD_COPIED_SUBJECT; + } + else + { + if (rc >= 0 || rc == PCRE2_ERROR_PARTIAL) match_data->subject = subject; + } goto EXIT; } diff --git a/thirdparty/pcre2/src/pcre2_error.c b/thirdparty/pcre2/src/pcre2_error.c index 4b3b3f1bc0d..1d02cf14a3a 100644 --- a/thirdparty/pcre2/src/pcre2_error.c +++ b/thirdparty/pcre2/src/pcre2_error.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -71,7 +71,7 @@ static const unsigned char compile_error_texts[] = /* 5 */ "number too big in {} quantifier\0" "missing terminating ] for character class\0" - "invalid escape sequence in character class\0" + "escape sequence is invalid in character class\0" "range out of order in character class\0" "quantifier does not follow a repeatable item\0" /* 10 */ @@ -95,7 +95,7 @@ static const unsigned char compile_error_texts[] = /* 25 */ "lookbehind assertion is not fixed length\0" "a relative value of zero is not allowed\0" - "conditional group contains more than two branches\0" + "conditional subpattern contains more than two branches\0" "assertion expected after (?( or (?(?C)\0" "digit expected after (?+ or (?-\0" /* 30 */ @@ -113,21 +113,21 @@ static const unsigned char compile_error_texts[] = /* 40 */ "invalid escape sequence in (*VERB) name\0" "unrecognized character after (?P\0" - "syntax error in subpattern name (missing terminator)\0" + "syntax error in subpattern name (missing terminator?)\0" "two named subpatterns have the same name (PCRE2_DUPNAMES not set)\0" - "group name must start with a non-digit\0" + "subpattern name must start with a non-digit\0" /* 45 */ "this version of PCRE2 does not have support for \\P, \\p, or \\X\0" "malformed \\P or \\p sequence\0" "unknown property name after \\P or \\p\0" - "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " characters)\0" + "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " code units)\0" "too many named subpatterns (maximum " XSTRING(MAX_NAME_COUNT) ")\0" /* 50 */ "invalid range in character class\0" "octal value is greater than \\377 in 8-bit non-UTF-8 mode\0" "internal error: overran compiling workspace\0" "internal error: previously-checked referenced subpattern not found\0" - "DEFINE group contains more than one branch\0" + "DEFINE subpattern contains more than one branch\0" /* 55 */ "missing opening brace after \\o\0" "internal error: unknown newline setting\0" @@ -137,7 +137,7 @@ static const unsigned char compile_error_texts[] = "obsolete error (should not occur)\0" /* Was the above */ /* 60 */ "(*VERB) not recognized or malformed\0" - "group number is too big\0" + "subpattern number is too big\0" "subpattern name expected\0" "internal error: parsed pattern overflow\0" "non-octal character in \\o{} (closing brace missing?)\0" @@ -181,6 +181,9 @@ static const unsigned char compile_error_texts[] = "invalid option bits with PCRE2_LITERAL\0" "\\N{U+dddd} is supported only in Unicode (UTF) mode\0" "invalid hyphen in option setting\0" + /* 95 */ + "(*alpha_assertion) not recognized\0" + "script runs require Unicode support, which this version of PCRE2 does not have\0" ; /* Match-time and UTF error texts are in the same format. */ diff --git a/thirdparty/pcre2/src/pcre2_extuni.c b/thirdparty/pcre2/src/pcre2_extuni.c index 237211abf7f..5a719e9cb4f 100644 --- a/thirdparty/pcre2/src/pcre2_extuni.c +++ b/thirdparty/pcre2/src/pcre2_extuni.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -100,7 +100,7 @@ while (eptr < end_subject) int len = 1; if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + if ((PRIV(ucp_gbtable)[lgb] & (1u << rgb)) == 0) break; /* Not breaking between Regional Indicators is allowed only if there are an even number of preceding RIs. */ diff --git a/thirdparty/pcre2/src/pcre2_internal.h b/thirdparty/pcre2/src/pcre2_internal.h index 8750f2f1746..814d91bddb5 100644 --- a/thirdparty/pcre2/src/pcre2_internal.h +++ b/thirdparty/pcre2/src/pcre2_internal.h @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -148,16 +148,7 @@ pcre2_match() because of the way it backtracks. */ /* When checking for integer overflow in pcre2_compile(), we need to handle large integers. If a 64-bit integer type is available, we can use that. Otherwise we have to cast to double, which of course requires floating point -arithmetic. Handle this by defining a macro for the appropriate type. If -stdint.h is available, include it; it may define INT64_MAX. Systems that do not -have stdint.h (e.g. Solaris) may have inttypes.h. The macro int64_t may be set -by "configure". */ - -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#endif +arithmetic. Handle this by defining a macro for the appropriate type. */ #if defined INT64_MAX || defined int64_t #define INT64_OR_DOUBLE int64_t @@ -535,6 +526,10 @@ enum { PCRE2_MATCHEDBY_INTERPRETER, /* pcre2_match() */ PCRE2_MATCHEDBY_DFA_INTERPRETER, /* pcre2_dfa_match() */ PCRE2_MATCHEDBY_JIT }; /* pcre2_jit_match() */ +/* Values for the flags field in a match data block. */ + +#define PCRE2_MD_COPIED_SUBJECT 0x01u + /* Magic number to provide a small check against being handed junk. */ #define MAGIC_NUMBER 0x50435245UL /* 'PCRE' */ @@ -569,11 +564,11 @@ these tables. */ without checking pcre2_jit_compile.c, which has an assertion to ensure that ctype_word has the value 16. */ -#define ctype_space 0x01 -#define ctype_letter 0x02 -#define ctype_digit 0x04 -#define ctype_xdigit 0x08 /* not actually used any more */ -#define ctype_word 0x10 /* alphanumeric or '_' */ +#define ctype_space 0x01 +#define ctype_letter 0x02 +#define ctype_lcletter 0x04 +#define ctype_digit 0x08 +#define ctype_word 0x10 /* alphanumeric or '_' */ /* Offsets of the various tables from the base tables pointer, and total length of the tables. */ @@ -874,34 +869,48 @@ a positive value. */ #define STR_RIGHT_CURLY_BRACKET "}" #define STR_TILDE "~" -#define STRING_ACCEPT0 "ACCEPT\0" -#define STRING_COMMIT0 "COMMIT\0" -#define STRING_F0 "F\0" -#define STRING_FAIL0 "FAIL\0" -#define STRING_MARK0 "MARK\0" -#define STRING_PRUNE0 "PRUNE\0" -#define STRING_SKIP0 "SKIP\0" -#define STRING_THEN "THEN" +#define STRING_ACCEPT0 "ACCEPT\0" +#define STRING_COMMIT0 "COMMIT\0" +#define STRING_F0 "F\0" +#define STRING_FAIL0 "FAIL\0" +#define STRING_MARK0 "MARK\0" +#define STRING_PRUNE0 "PRUNE\0" +#define STRING_SKIP0 "SKIP\0" +#define STRING_THEN "THEN" -#define STRING_alpha0 "alpha\0" -#define STRING_lower0 "lower\0" -#define STRING_upper0 "upper\0" -#define STRING_alnum0 "alnum\0" -#define STRING_ascii0 "ascii\0" -#define STRING_blank0 "blank\0" -#define STRING_cntrl0 "cntrl\0" -#define STRING_digit0 "digit\0" -#define STRING_graph0 "graph\0" -#define STRING_print0 "print\0" -#define STRING_punct0 "punct\0" -#define STRING_space0 "space\0" -#define STRING_word0 "word\0" -#define STRING_xdigit "xdigit" +#define STRING_atomic0 "atomic\0" +#define STRING_pla0 "pla\0" +#define STRING_plb0 "plb\0" +#define STRING_nla0 "nla\0" +#define STRING_nlb0 "nlb\0" +#define STRING_sr0 "sr\0" +#define STRING_asr0 "asr\0" +#define STRING_positive_lookahead0 "positive_lookahead\0" +#define STRING_positive_lookbehind0 "positive_lookbehind\0" +#define STRING_negative_lookahead0 "negative_lookahead\0" +#define STRING_negative_lookbehind0 "negative_lookbehind\0" +#define STRING_script_run0 "script_run\0" +#define STRING_atomic_script_run "atomic_script_run" -#define STRING_DEFINE "DEFINE" -#define STRING_VERSION "VERSION" -#define STRING_WEIRD_STARTWORD "[:<:]]" -#define STRING_WEIRD_ENDWORD "[:>:]]" +#define STRING_alpha0 "alpha\0" +#define STRING_lower0 "lower\0" +#define STRING_upper0 "upper\0" +#define STRING_alnum0 "alnum\0" +#define STRING_ascii0 "ascii\0" +#define STRING_blank0 "blank\0" +#define STRING_cntrl0 "cntrl\0" +#define STRING_digit0 "digit\0" +#define STRING_graph0 "graph\0" +#define STRING_print0 "print\0" +#define STRING_punct0 "punct\0" +#define STRING_space0 "space\0" +#define STRING_word0 "word\0" +#define STRING_xdigit "xdigit" + +#define STRING_DEFINE "DEFINE" +#define STRING_VERSION "VERSION" +#define STRING_WEIRD_STARTWORD "[:<:]]" +#define STRING_WEIRD_ENDWORD "[:>:]]" #define STRING_CR_RIGHTPAR "CR)" #define STRING_LF_RIGHTPAR "LF)" @@ -1150,34 +1159,48 @@ only. */ #define STR_RIGHT_CURLY_BRACKET "\175" #define STR_TILDE "\176" -#define STRING_ACCEPT0 STR_A STR_C STR_C STR_E STR_P STR_T "\0" -#define STRING_COMMIT0 STR_C STR_O STR_M STR_M STR_I STR_T "\0" -#define STRING_F0 STR_F "\0" -#define STRING_FAIL0 STR_F STR_A STR_I STR_L "\0" -#define STRING_MARK0 STR_M STR_A STR_R STR_K "\0" -#define STRING_PRUNE0 STR_P STR_R STR_U STR_N STR_E "\0" -#define STRING_SKIP0 STR_S STR_K STR_I STR_P "\0" -#define STRING_THEN STR_T STR_H STR_E STR_N +#define STRING_ACCEPT0 STR_A STR_C STR_C STR_E STR_P STR_T "\0" +#define STRING_COMMIT0 STR_C STR_O STR_M STR_M STR_I STR_T "\0" +#define STRING_F0 STR_F "\0" +#define STRING_FAIL0 STR_F STR_A STR_I STR_L "\0" +#define STRING_MARK0 STR_M STR_A STR_R STR_K "\0" +#define STRING_PRUNE0 STR_P STR_R STR_U STR_N STR_E "\0" +#define STRING_SKIP0 STR_S STR_K STR_I STR_P "\0" +#define STRING_THEN STR_T STR_H STR_E STR_N -#define STRING_alpha0 STR_a STR_l STR_p STR_h STR_a "\0" -#define STRING_lower0 STR_l STR_o STR_w STR_e STR_r "\0" -#define STRING_upper0 STR_u STR_p STR_p STR_e STR_r "\0" -#define STRING_alnum0 STR_a STR_l STR_n STR_u STR_m "\0" -#define STRING_ascii0 STR_a STR_s STR_c STR_i STR_i "\0" -#define STRING_blank0 STR_b STR_l STR_a STR_n STR_k "\0" -#define STRING_cntrl0 STR_c STR_n STR_t STR_r STR_l "\0" -#define STRING_digit0 STR_d STR_i STR_g STR_i STR_t "\0" -#define STRING_graph0 STR_g STR_r STR_a STR_p STR_h "\0" -#define STRING_print0 STR_p STR_r STR_i STR_n STR_t "\0" -#define STRING_punct0 STR_p STR_u STR_n STR_c STR_t "\0" -#define STRING_space0 STR_s STR_p STR_a STR_c STR_e "\0" -#define STRING_word0 STR_w STR_o STR_r STR_d "\0" -#define STRING_xdigit STR_x STR_d STR_i STR_g STR_i STR_t +#define STRING_atomic0 STR_a STR_t STR_o STR_m STR_i STR_c "\0" +#define STRING_pla0 STR_p STR_l STR_a "\0" +#define STRING_plb0 STR_p STR_l STR_b "\0" +#define STRING_nla0 STR_n STR_l STR_a "\0" +#define STRING_nlb0 STR_n STR_l STR_b "\0" +#define STRING_sr0 STR_s STR_r "\0" +#define STRING_asr0 STR_a STR_s STR_r "\0" +#define STRING_positive_lookahead0 STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_a STR_h STR_e STR_a STR_d "\0" +#define STRING_positive_lookbehind0 STR_p STR_o STR_s STR_i STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_b STR_e STR_h STR_i STR_n STR_d "\0" +#define STRING_negative_lookahead0 STR_n STR_e STR_g STR_a STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_a STR_h STR_e STR_a STR_d "\0" +#define STRING_negative_lookbehind0 STR_n STR_e STR_g STR_a STR_t STR_i STR_v STR_e STR_UNDERSCORE STR_l STR_o STR_o STR_k STR_b STR_e STR_h STR_i STR_n STR_d "\0" +#define STRING_script_run0 STR_s STR_c STR_r STR_i STR_p STR_t STR_UNDERSCORE STR_r STR_u STR_n "\0" +#define STRING_atomic_script_run STR_a STR_t STR_o STR_m STR_i STR_c STR_UNDERSCORE STR_s STR_c STR_r STR_i STR_p STR_t STR_UNDERSCORE STR_r STR_u STR_n -#define STRING_DEFINE STR_D STR_E STR_F STR_I STR_N STR_E -#define STRING_VERSION STR_V STR_E STR_R STR_S STR_I STR_O STR_N -#define STRING_WEIRD_STARTWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_LESS_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET -#define STRING_WEIRD_ENDWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_GREATER_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET +#define STRING_alpha0 STR_a STR_l STR_p STR_h STR_a "\0" +#define STRING_lower0 STR_l STR_o STR_w STR_e STR_r "\0" +#define STRING_upper0 STR_u STR_p STR_p STR_e STR_r "\0" +#define STRING_alnum0 STR_a STR_l STR_n STR_u STR_m "\0" +#define STRING_ascii0 STR_a STR_s STR_c STR_i STR_i "\0" +#define STRING_blank0 STR_b STR_l STR_a STR_n STR_k "\0" +#define STRING_cntrl0 STR_c STR_n STR_t STR_r STR_l "\0" +#define STRING_digit0 STR_d STR_i STR_g STR_i STR_t "\0" +#define STRING_graph0 STR_g STR_r STR_a STR_p STR_h "\0" +#define STRING_print0 STR_p STR_r STR_i STR_n STR_t "\0" +#define STRING_punct0 STR_p STR_u STR_n STR_c STR_t "\0" +#define STRING_space0 STR_s STR_p STR_a STR_c STR_e "\0" +#define STRING_word0 STR_w STR_o STR_r STR_d "\0" +#define STRING_xdigit STR_x STR_d STR_i STR_g STR_i STR_t + +#define STRING_DEFINE STR_D STR_E STR_F STR_I STR_N STR_E +#define STRING_VERSION STR_V STR_E STR_R STR_S STR_I STR_O STR_N +#define STRING_WEIRD_STARTWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_LESS_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET +#define STRING_WEIRD_ENDWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_GREATER_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET #define STRING_CR_RIGHTPAR STR_C STR_R STR_RIGHT_PARENTHESIS #define STRING_LF_RIGHTPAR STR_L STR_F STR_RIGHT_PARENTHESIS @@ -1485,70 +1508,71 @@ enum { OP_ASSERTBACK, /* 128 Positive lookbehind */ OP_ASSERTBACK_NOT, /* 129 Negative lookbehind */ - /* ONCE, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come immediately after the - assertions, with ONCE first, as there's a test for >= ONCE for a subpattern - that isn't an assertion. The POS versions must immediately follow the non-POS - versions in each case. */ + /* ONCE, SCRIPT_RUN, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come + immediately after the assertions, with ONCE first, as there's a test for >= + ONCE for a subpattern that isn't an assertion. The POS versions must + immediately follow the non-POS versions in each case. */ OP_ONCE, /* 130 Atomic group, contains captures */ - OP_BRA, /* 131 Start of non-capturing bracket */ - OP_BRAPOS, /* 132 Ditto, with unlimited, possessive repeat */ - OP_CBRA, /* 133 Start of capturing bracket */ - OP_CBRAPOS, /* 134 Ditto, with unlimited, possessive repeat */ - OP_COND, /* 135 Conditional group */ + OP_SCRIPT_RUN, /* 131 Non-capture, but check characters' scripts */ + OP_BRA, /* 132 Start of non-capturing bracket */ + OP_BRAPOS, /* 133 Ditto, with unlimited, possessive repeat */ + OP_CBRA, /* 134 Start of capturing bracket */ + OP_CBRAPOS, /* 135 Ditto, with unlimited, possessive repeat */ + OP_COND, /* 136 Conditional group */ /* These five must follow the previous five, in the same order. There's a check for >= SBRA to distinguish the two sets. */ - OP_SBRA, /* 136 Start of non-capturing bracket, check empty */ - OP_SBRAPOS, /* 137 Ditto, with unlimited, possessive repeat */ - OP_SCBRA, /* 138 Start of capturing bracket, check empty */ - OP_SCBRAPOS, /* 139 Ditto, with unlimited, possessive repeat */ - OP_SCOND, /* 140 Conditional group, check empty */ + OP_SBRA, /* 137 Start of non-capturing bracket, check empty */ + OP_SBRAPOS, /* 138 Ditto, with unlimited, possessive repeat */ + OP_SCBRA, /* 139 Start of capturing bracket, check empty */ + OP_SCBRAPOS, /* 140 Ditto, with unlimited, possessive repeat */ + OP_SCOND, /* 141 Conditional group, check empty */ /* The next two pairs must (respectively) be kept together. */ - OP_CREF, /* 141 Used to hold a capture number as condition */ - OP_DNCREF, /* 142 Used to point to duplicate names as a condition */ - OP_RREF, /* 143 Used to hold a recursion number as condition */ - OP_DNRREF, /* 144 Used to point to duplicate names as a condition */ - OP_FALSE, /* 145 Always false (used by DEFINE and VERSION) */ - OP_TRUE, /* 146 Always true (used by VERSION) */ + OP_CREF, /* 142 Used to hold a capture number as condition */ + OP_DNCREF, /* 143 Used to point to duplicate names as a condition */ + OP_RREF, /* 144 Used to hold a recursion number as condition */ + OP_DNRREF, /* 145 Used to point to duplicate names as a condition */ + OP_FALSE, /* 146 Always false (used by DEFINE and VERSION) */ + OP_TRUE, /* 147 Always true (used by VERSION) */ - OP_BRAZERO, /* 147 These two must remain together and in this */ - OP_BRAMINZERO, /* 148 order. */ - OP_BRAPOSZERO, /* 149 */ + OP_BRAZERO, /* 148 These two must remain together and in this */ + OP_BRAMINZERO, /* 149 order. */ + OP_BRAPOSZERO, /* 150 */ /* These are backtracking control verbs */ - OP_MARK, /* 150 always has an argument */ - OP_PRUNE, /* 151 */ - OP_PRUNE_ARG, /* 152 same, but with argument */ - OP_SKIP, /* 153 */ - OP_SKIP_ARG, /* 154 same, but with argument */ - OP_THEN, /* 155 */ - OP_THEN_ARG, /* 156 same, but with argument */ - OP_COMMIT, /* 157 */ - OP_COMMIT_ARG, /* 158 same, but with argument */ + OP_MARK, /* 151 always has an argument */ + OP_PRUNE, /* 152 */ + OP_PRUNE_ARG, /* 153 same, but with argument */ + OP_SKIP, /* 154 */ + OP_SKIP_ARG, /* 155 same, but with argument */ + OP_THEN, /* 156 */ + OP_THEN_ARG, /* 157 same, but with argument */ + OP_COMMIT, /* 158 */ + OP_COMMIT_ARG, /* 159 same, but with argument */ /* These are forced failure and success verbs. FAIL and ACCEPT do accept an argument, but these cases can be compiled as, for example, (*MARK:X)(*FAIL) without the need for a special opcode. */ - OP_FAIL, /* 159 */ - OP_ACCEPT, /* 160 */ - OP_ASSERT_ACCEPT, /* 161 Used inside assertions */ - OP_CLOSE, /* 162 Used before OP_ACCEPT to close open captures */ + OP_FAIL, /* 160 */ + OP_ACCEPT, /* 161 */ + OP_ASSERT_ACCEPT, /* 162 Used inside assertions */ + OP_CLOSE, /* 163 Used before OP_ACCEPT to close open captures */ /* This is used to skip a subpattern with a {0} quantifier */ - OP_SKIPZERO, /* 163 */ + OP_SKIPZERO, /* 164 */ /* This is used to identify a DEFINE group during compilation so that it can be checked for having only one branch. It is changed to OP_FALSE before compilation finishes. */ - OP_DEFINE, /* 164 */ + OP_DEFINE, /* 165 */ /* This is not an opcode, but is used to check that tables indexed by opcode are the correct length, in order to catch updating errors - there have been @@ -1596,6 +1620,7 @@ some cases doesn't actually use these names at all). */ "Alt", "Ket", "KetRmax", "KetRmin", "KetRpos", \ "Reverse", "Assert", "Assert not", "AssertB", "AssertB not", \ "Once", \ + "Script run", \ "Bra", "BraPos", "CBra", "CBraPos", \ "Cond", \ "SBra", "SBraPos", "SCBra", "SCBraPos", \ @@ -1679,6 +1704,7 @@ in UTF-8 mode. The code that uses this table must know about such things. */ 1+LINK_SIZE, /* Assert behind */ \ 1+LINK_SIZE, /* Assert behind not */ \ 1+LINK_SIZE, /* ONCE */ \ + 1+LINK_SIZE, /* SCRIPT_RUN */ \ 1+LINK_SIZE, /* BRA */ \ 1+LINK_SIZE, /* BRAPOS */ \ 1+LINK_SIZE+IMM2_SIZE, /* CBRA */ \ @@ -1747,6 +1773,8 @@ typedef struct { uint8_t gbprop; /* ucp_gbControl, etc. (grapheme break property) */ uint8_t caseset; /* offset to multichar other cases or zero */ int32_t other_case; /* offset to other case, or zero if none */ + int16_t scriptx; /* script extension value */ + int16_t dummy; /* spare - to round to multiple of 4 bytes */ } ucd_record; /* UCD access macros */ @@ -1769,6 +1797,7 @@ typedef struct { #define UCD_GRAPHBREAK(ch) GET_UCD(ch)->gbprop #define UCD_CASESET(ch) GET_UCD(ch)->caseset #define UCD_OTHERCASE(ch) ((uint32_t)((int)ch + (int)(GET_UCD(ch)->other_case))) +#define UCD_SCRIPTX(ch) GET_UCD(ch)->scriptx /* Header for serialized pcre2 codes. */ @@ -1826,6 +1855,8 @@ extern const uint8_t PRIV(utf8_table4)[]; #define _pcre2_hspace_list PCRE2_SUFFIX(_pcre2_hspace_list_) #define _pcre2_vspace_list PCRE2_SUFFIX(_pcre2_vspace_list_) #define _pcre2_ucd_caseless_sets PCRE2_SUFFIX(_pcre2_ucd_caseless_sets_) +#define _pcre2_ucd_digit_sets PCRE2_SUFFIX(_pcre2_ucd_digit_sets_) +#define _pcre2_ucd_script_sets PCRE2_SUFFIX(_pcre2_ucd_script_sets_) #define _pcre2_ucd_records PCRE2_SUFFIX(_pcre2_ucd_records_) #define _pcre2_ucd_stage1 PCRE2_SUFFIX(_pcre2_ucd_stage1_) #define _pcre2_ucd_stage2 PCRE2_SUFFIX(_pcre2_ucd_stage2_) @@ -1847,6 +1878,8 @@ extern const uint8_t PRIV(default_tables)[]; extern const uint32_t PRIV(hspace_list)[]; extern const uint32_t PRIV(vspace_list)[]; extern const uint32_t PRIV(ucd_caseless_sets)[]; +extern const uint32_t PRIV(ucd_digit_sets)[]; +extern const uint8_t PRIV(ucd_script_sets)[]; extern const ucd_record PRIV(ucd_records)[]; #if PCRE2_CODE_UNIT_WIDTH == 32 extern const ucd_record PRIV(dummy_ucd_record)[]; @@ -1894,6 +1927,7 @@ is available. */ #define _pcre2_jit_get_target PCRE2_SUFFIX(_pcre2_jit_get_target_) #define _pcre2_memctl_malloc PCRE2_SUFFIX(_pcre2_memctl_malloc_) #define _pcre2_ord2utf PCRE2_SUFFIX(_pcre2_ord2utf_) +#define _pcre2_script_run PCRE2_SUFFIX(_pcre2_script_run_) #define _pcre2_strcmp PCRE2_SUFFIX(_pcre2_strcmp_) #define _pcre2_strcmp_c8 PCRE2_SUFFIX(_pcre2_strcmp_c8_) #define _pcre2_strcpy_c8 PCRE2_SUFFIX(_pcre2_strcpy_c8_) @@ -1908,7 +1942,7 @@ is available. */ extern int _pcre2_auto_possessify(PCRE2_UCHAR *, BOOL, const compile_block *); extern int _pcre2_check_escape(PCRE2_SPTR *, PCRE2_SPTR, uint32_t *, - int *, uint32_t, BOOL, compile_block *); + int *, uint32_t, uint32_t, BOOL, compile_block *); extern PCRE2_SPTR _pcre2_extuni(uint32_t, PCRE2_SPTR, PCRE2_SPTR, PCRE2_SPTR, BOOL, int *); extern PCRE2_SPTR _pcre2_find_bracket(PCRE2_SPTR, BOOL, int); @@ -1920,6 +1954,7 @@ extern size_t _pcre2_jit_get_size(void *); const char * _pcre2_jit_get_target(void); extern void * _pcre2_memctl_malloc(size_t, pcre2_memctl *); extern unsigned int _pcre2_ord2utf(uint32_t, PCRE2_UCHAR *); +extern BOOL _pcre2_script_run(PCRE2_SPTR, PCRE2_SPTR, BOOL); extern int _pcre2_strcmp(PCRE2_SPTR, PCRE2_SPTR); extern int _pcre2_strcmp_c8(PCRE2_SPTR, const char *); extern PCRE2_SIZE _pcre2_strcpy_c8(PCRE2_UCHAR *, const char *); diff --git a/thirdparty/pcre2/src/pcre2_intmodedep.h b/thirdparty/pcre2/src/pcre2_intmodedep.h index 62626d0a8a7..bf3a235984d 100644 --- a/thirdparty/pcre2/src/pcre2_intmodedep.h +++ b/thirdparty/pcre2/src/pcre2_intmodedep.h @@ -585,6 +585,8 @@ typedef struct pcre2_real_match_context { #endif int (*callout)(pcre2_callout_block *, void *); void *callout_data; + int (*substitute_callout)(pcre2_substitute_callout_block *, void *); + void *substitute_callout_data; PCRE2_SIZE offset_limit; uint32_t heap_limit; uint32_t match_limit; @@ -656,7 +658,8 @@ typedef struct pcre2_real_match_data { PCRE2_SIZE leftchar; /* Offset to leftmost code unit */ PCRE2_SIZE rightchar; /* Offset to rightmost code unit */ PCRE2_SIZE startchar; /* Offset to starting code unit */ - uint16_t matchedby; /* Type of match (normal, JIT, DFA) */ + uint8_t matchedby; /* Type of match (normal, JIT, DFA) */ + uint8_t flags; /* Various flags */ uint16_t oveccount; /* Number of pairs */ int rc; /* The return code from the match */ PCRE2_SIZE ovector[131072]; /* Must be last in the structure */ diff --git a/thirdparty/pcre2/src/pcre2_jit_compile.c b/thirdparty/pcre2/src/pcre2_jit_compile.c index 32e985b7938..1f21bfb6ada 100644 --- a/thirdparty/pcre2/src/pcre2_jit_compile.c +++ b/thirdparty/pcre2/src/pcre2_jit_compile.c @@ -477,12 +477,22 @@ typedef struct compiler_common { BOOL alt_circumflex; #ifdef SUPPORT_UNICODE BOOL utf; + BOOL invalid_utf; BOOL use_ucp; + /* Points to saving area for iref. */ + sljit_s32 iref_ptr; jump_list *getucd; + jump_list *getucdtype; #if PCRE2_CODE_UNIT_WIDTH == 8 jump_list *utfreadchar; - jump_list *utfreadchar16; jump_list *utfreadtype8; + jump_list *utfpeakcharback; +#endif +#if PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16 + jump_list *utfreadchar_invalid; + jump_list *utfreadnewline_invalid; + jump_list *utfmoveback_invalid; + jump_list *utfpeakcharback_invalid; #endif #endif /* SUPPORT_UNICODE */ } compiler_common; @@ -616,7 +626,183 @@ the start pointers when the end of the capturing group has not yet reached. */ #define READ_CHAR_MAX 0x7fffffff -#define INVALID_UTF_CHAR 888 +#define INVALID_UTF_CHAR -1 +#define UNASSIGNED_UTF_CHAR 888 + +#if defined SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 + +#define GETCHARINC_INVALID(c, ptr, end, invalid_action) \ + { \ + if (ptr[0] <= 0x7f) \ + c = *ptr++; \ + else if (ptr + 1 < end && ptr[1] >= 0x80 && ptr[1] < 0xc0) \ + { \ + c = ptr[1] - 0x80; \ + \ + if (ptr[0] >= 0xc2 && ptr[0] <= 0xdf) \ + { \ + c |= (ptr[0] - 0xc0) << 6; \ + ptr += 2; \ + } \ + else if (ptr + 2 < end && ptr[2] >= 0x80 && ptr[2] < 0xc0) \ + { \ + c = c << 6 | (ptr[2] - 0x80); \ + \ + if (ptr[0] >= 0xe0 && ptr[0] <= 0xef) \ + { \ + c |= (ptr[0] - 0xe0) << 12; \ + ptr += 3; \ + \ + if (c < 0x800 || (c >= 0xd800 && c < 0xe000)) \ + { \ + invalid_action; \ + } \ + } \ + else if (ptr + 3 < end && ptr[3] >= 0x80 && ptr[3] < 0xc0) \ + { \ + c = c << 6 | (ptr[3] - 0x80); \ + \ + if (ptr[0] >= 0xf0 && ptr[0] <= 0xf4) \ + { \ + c |= (ptr[0] - 0xf0) << 18; \ + ptr += 4; \ + \ + if (c >= 0x110000 || c < 0x10000) \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } + +#define GETCHARBACK_INVALID(c, ptr, start, invalid_action) \ + { \ + if (ptr[-1] <= 0x7f) \ + c = *ptr--; \ + else if (ptr - 1 > start && ptr[-1] >= 0x80 && ptr[-1] < 0xc0) \ + { \ + c = ptr[-1] - 0x80; \ + \ + if (ptr[-2] >= 0xc2 && ptr[-2] <= 0xdf) \ + { \ + c |= (ptr[-2] - 0xc0) << 6; \ + ptr -= 2; \ + } \ + else if (ptr - 2 > start && ptr[-2] >= 0x80 && ptr[-2] < 0xc0) \ + { \ + c = c << 6 | (ptr[-2] - 0x80); \ + \ + if (ptr[-3] >= 0xe0 && ptr[-3] <= 0xef) \ + { \ + c |= (ptr[-3] - 0xe0) << 12; \ + ptr -= 3; \ + \ + if (c < 0x800 || (c >= 0xd800 && c < 0xe000)) \ + { \ + invalid_action; \ + } \ + } \ + else if (ptr - 3 > start && ptr[-3] >= 0x80 && ptr[-3] < 0xc0) \ + { \ + c = c << 6 | (ptr[-3] - 0x80); \ + \ + if (ptr[-4] >= 0xf0 && ptr[-4] <= 0xf4) \ + { \ + c |= (ptr[-4] - 0xf0) << 18; \ + ptr -= 4; \ + \ + if (c >= 0x110000 || c < 0x10000) \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } \ + else \ + { \ + invalid_action; \ + } \ + } + +#elif PCRE2_CODE_UNIT_WIDTH == 16 + +#define GETCHARINC_INVALID(c, ptr, end, invalid_action) \ + { \ + if (ptr[0] < 0xd800 || ptr[0] >= 0xe000) \ + c = *ptr++; \ + else if (ptr[0] < 0xdc00 && ptr + 1 < end && ptr[1] >= 0xdc00 && ptr[1] < 0xe000) \ + { \ + c = (((ptr[0] - 0xd800) << 10) | (ptr[1] - 0xdc00)) + 0x10000; \ + ptr += 2; \ + } \ + else \ + { \ + invalid_action; \ + } \ + } + +#define GETCHARBACK_INVALID(c, ptr, start, invalid_action) \ + { \ + if (ptr[-1] < 0xd800 || ptr[-1] >= 0xe000) \ + c = *ptr--; \ + else if (ptr[-1] >= 0xdc00 && ptr - 1 > start && ptr[-2] >= 0xd800 && ptr[-2] < 0xdc00) \ + { \ + c = (((ptr[-2] - 0xd800) << 10) | (ptr[-1] - 0xdc00)) + 0x10000; \ + ptr -= 2; \ + } \ + else \ + { \ + invalid_action; \ + } \ + } + + +#elif PCRE2_CODE_UNIT_WIDTH == 32 + +#define GETCHARINC_INVALID(c, ptr, end, invalid_action) \ + { \ + if (ptr[0] < 0x110000) \ + c = *ptr++; \ + else \ + { \ + invalid_action; \ + } \ + } + +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ +#endif /* SUPPORT_UNICODE */ static PCRE2_SPTR bracketend(PCRE2_SPTR cc) { @@ -716,6 +902,7 @@ switch(*cc) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRA: case OP_BRAPOS: case OP_CBRA: @@ -869,8 +1056,16 @@ while (cc < ccend) cc += 1; break; - case OP_REF: case OP_REFI: +#ifdef SUPPORT_UNICODE + if (common->iref_ptr == 0) + { + common->iref_ptr = common->ovector_start; + common->ovector_start += 3 * sizeof(sljit_sw); + } +#endif /* SUPPORT_UNICODE */ + /* Fall through. */ + case OP_REF: common->optimized_cbracket[GET2(cc, 1)] = 0; cc += 1 + IMM2_SIZE; break; @@ -1375,6 +1570,7 @@ while (cc < ccend) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRAPOS: case OP_SBRA: case OP_SBRAPOS: @@ -1951,6 +2147,7 @@ while (cc < ccend) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRAPOS: case OP_SBRA: case OP_SBRAPOS: @@ -2174,14 +2371,14 @@ if (base_reg != TMP2) else { status.saved_tmp_regs[1] = RETURN_ADDR; - if (sljit_get_register_index (RETURN_ADDR) == -1) + if (sljit_get_register_index(RETURN_ADDR) == -1) status.tmp_regs[1] = STR_PTR; else status.tmp_regs[1] = RETURN_ADDR; } status.saved_tmp_regs[2] = TMP3; -if (sljit_get_register_index (TMP3) == -1) +if (sljit_get_register_index(TMP3) == -1) status.tmp_regs[2] = STR_END; else status.tmp_regs[2] = TMP3; @@ -2274,6 +2471,7 @@ while (cc < ccend) case OP_ASSERTBACK: case OP_ASSERTBACK_NOT: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRAPOS: case OP_SBRA: case OP_SBRAPOS: @@ -3059,13 +3257,13 @@ return (0 << 8) | bit; #ifdef SUPPORT_UNICODE if (common->utf && c > 65535) { - if (bit >= (1 << 10)) + if (bit >= (1u << 10)) bit >>= 10; else return (bit < 256) ? ((2 << 8) | bit) : ((3 << 8) | (bit >> 8)); } #endif /* SUPPORT_UNICODE */ -return (bit < 256) ? ((0 << 8) | bit) : ((1 << 8) | (bit >> 8)); +return (bit < 256) ? ((0u << 8) | bit) : ((1u << 8) | (bit >> 8)); #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ } @@ -3159,47 +3357,335 @@ else JUMPHERE(jump); } -static void peek_char(compiler_common *common, sljit_u32 max) +static void peek_char(compiler_common *common, sljit_u32 max, sljit_s32 dst, sljit_sw dstw, jump_list **backtracks) { /* Reads the character into TMP1, keeps STR_PTR. -Does not check STR_END. TMP2 Destroyed. */ +Does not check STR_END. TMP2, dst, RETURN_ADDR Destroyed. */ DEFINE_COMPILER; #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 struct sljit_jump *jump; -#endif +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ SLJIT_UNUSED_ARG(max); +SLJIT_UNUSED_ARG(dst); +SLJIT_UNUSED_ARG(dstw); +SLJIT_UNUSED_ARG(backtracks); -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf) { if (max < 128) return; - jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x80); + OP1(SLJIT_MOV, dst, dstw, STR_PTR, 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + add_jump(compiler, common->invalid_utf ? &common->utfreadchar_invalid : &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, STR_PTR, 0, dst, dstw); + if (backtracks && common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); JUMPHERE(jump); } -#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16 +#elif PCRE2_CODE_UNIT_WIDTH == 16 if (common->utf) { if (max < 0xd800) return; OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - /* TMP2 contains the high surrogate. */ - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + + if (common->invalid_utf) + { + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800); + OP1(SLJIT_MOV, dst, dstw, STR_PTR, 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, STR_PTR, 0, dst, dstw); + if (backtracks && common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + } + else + { + /* TMP2 contains the high surrogate. */ + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000 - 0xdc00); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + } + JUMPHERE(jump); } +#elif PCRE2_CODE_UNIT_WIDTH == 32 +if (common->invalid_utf) + { + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + else + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x110000); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + } + } +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ +#endif /* SUPPORT_UNICODE */ +} + +static void peek_char_back(compiler_common *common, sljit_u32 max, jump_list **backtracks) +{ +/* Reads one character back without moving STR_PTR. TMP2 must +contain the start of the subject buffer. Affects TMP1, TMP2, and RETURN_ADDR. */ +DEFINE_COMPILER; + +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +struct sljit_jump *jump; +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ + +SLJIT_UNUSED_ARG(max); +SLJIT_UNUSED_ARG(backtracks); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 +if (common->utf) + { + if (max < 128) return; + + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x80); + if (common->invalid_utf) + { + add_jump(compiler, &common->utfpeakcharback_invalid, JUMP(SLJIT_FAST_CALL)); + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + } + else + add_jump(compiler, &common->utfpeakcharback, JUMP(SLJIT_FAST_CALL)); + JUMPHERE(jump); + } +#elif PCRE2_CODE_UNIT_WIDTH == 16 +if (common->utf) + { + if (max < 0xd800) return; + + if (common->invalid_utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); + add_jump(compiler, &common->utfpeakcharback_invalid, JUMP(SLJIT_FAST_CALL)); + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + } + else + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xdc00); + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe000 - 0xdc00); + /* TMP2 contains the low surrogate. */ + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x10000); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 10); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + } + JUMPHERE(jump); + } +#elif PCRE2_CODE_UNIT_WIDTH == 32 + if (common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ +#endif /* SUPPORT_UNICODE */ +} + +#define READ_CHAR_UPDATE_STR_PTR 0x1 +#define READ_CHAR_UTF8_NEWLINE 0x2 +#define READ_CHAR_NEWLINE (READ_CHAR_UPDATE_STR_PTR | READ_CHAR_UTF8_NEWLINE) +#define READ_CHAR_VALID_UTF 0x4 + +static void read_char(compiler_common *common, sljit_u32 min, sljit_u32 max, + jump_list **backtracks, sljit_u32 options) +{ +/* Reads the precise value of a character into TMP1, if the character is +between min and max (c >= min && c <= max). Otherwise it returns with a value +outside the range. Does not check STR_END. */ +DEFINE_COMPILER; +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +struct sljit_jump *jump; #endif +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 +struct sljit_jump *jump2; +#endif + +SLJIT_UNUSED_ARG(min); +SLJIT_UNUSED_ARG(max); +SLJIT_UNUSED_ARG(backtracks); +SLJIT_UNUSED_ARG(options); +SLJIT_ASSERT(min <= max); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +#ifdef SUPPORT_UNICODE +#if PCRE2_CODE_UNIT_WIDTH == 8 +if (common->utf) + { + if (max < 128 && !(options & READ_CHAR_UPDATE_STR_PTR)) return; + + if (common->invalid_utf && !(options & READ_CHAR_VALID_UTF)) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x80); + + if (options & READ_CHAR_UTF8_NEWLINE) + add_jump(compiler, &common->utfreadnewline_invalid, JUMP(SLJIT_FAST_CALL)); + else + add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL)); + + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + JUMPHERE(jump); + return; + } + + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + if (min >= 0x10000) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xf0); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); + if (!(options & READ_CHAR_UPDATE_STR_PTR)) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump2); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + else if (min >= 0x800 && max <= 0xffff) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xe0); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xf); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + if (!(options & READ_CHAR_UPDATE_STR_PTR)) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump2); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + else if (max >= 0x800) + { + add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); + } + else if (max < 128) + { + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + } + else + { + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (!(options & READ_CHAR_UPDATE_STR_PTR)) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + else + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + JUMPHERE(jump); + } +#elif PCRE2_CODE_UNIT_WIDTH == 16 +if (common->utf) + { + if (max < 0xd800 && !(options & READ_CHAR_UPDATE_STR_PTR)) return; + + if (common->invalid_utf && !(options & READ_CHAR_VALID_UTF)) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800); + + if (options & READ_CHAR_UTF8_NEWLINE) + add_jump(compiler, &common->utfreadnewline_invalid, JUMP(SLJIT_FAST_CALL)); + else + add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL)); + + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + JUMPHERE(jump); + return; + } + + if (max >= 0x10000) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800); + /* TMP2 contains the high surrogate. */ + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000 - 0xdc00); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump); + return; + } + + /* Skip low surrogate if necessary. */ + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && sljit_get_register_index(RETURN_ADDR) >= 0) + { + if (options & READ_CHAR_UPDATE_STR_PTR) + OP2(SLJIT_ADD, RETURN_ADDR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x400); + if (options & READ_CHAR_UPDATE_STR_PTR) + CMOV(SLJIT_LESS, STR_PTR, RETURN_ADDR, 0); + if (max >= 0xd800) + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, 0x10000); + } + else + { + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400); + if (options & READ_CHAR_UPDATE_STR_PTR) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + if (max >= 0xd800) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); + JUMPHERE(jump); + } + } +#elif PCRE2_CODE_UNIT_WIDTH == 32 +if (common->invalid_utf) + { + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + else + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x110000); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + } + } +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ +#endif /* SUPPORT_UNICODE */ } #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 @@ -3221,7 +3707,7 @@ while (bitset < end); return TRUE; } -static void read_char7_type(compiler_common *common, BOOL full_read) +static void read_char7_type(compiler_common *common, jump_list **backtracks, BOOL negated) { /* Reads the precise character type of a character into TMP1, if the character is less than 128. Otherwise it returns with zero. Does not check STR_END. The @@ -3234,153 +3720,31 @@ SLJIT_ASSERT(common->utf); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +/* All values > 127 are zero in ctypes. */ OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); -if (full_read) +if (negated) { - jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x80); + + if (common->invalid_utf) + { + add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL)); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); + } + else + { + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + } JUMPHERE(jump); } } #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 */ -static void read_char_range(compiler_common *common, sljit_u32 min, sljit_u32 max, BOOL update_str_ptr) -{ -/* Reads the precise value of a character into TMP1, if the character is -between min and max (c >= min && c <= max). Otherwise it returns with a value -outside the range. Does not check STR_END. */ -DEFINE_COMPILER; -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -struct sljit_jump *jump; -#endif -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 -struct sljit_jump *jump2; -#endif - -SLJIT_UNUSED_ARG(update_str_ptr); -SLJIT_UNUSED_ARG(min); -SLJIT_UNUSED_ARG(max); -SLJIT_ASSERT(min <= max); - -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 -if (common->utf) - { - if (max < 128 && !update_str_ptr) return; - - jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - if (min >= 0x10000) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xf0); - if (update_str_ptr) - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump2); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - else if (min >= 0x800 && max <= 0xffff) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xe0); - if (update_str_ptr) - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xf); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump2); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - else if (max >= 0x800) - add_jump(compiler, (max < 0x10000) ? &common->utfreadchar16 : &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); - else if (max < 128) - { - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - } - else - { - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - else - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - JUMPHERE(jump); - } -#endif - -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16 -if (common->utf) - { - if (max >= 0x10000) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - /* TMP2 contains the high surrogate. */ - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump); - return; - } - - if (max < 0xd800 && !update_str_ptr) return; - - /* Skip low surrogate if necessary. */ - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - if (max >= 0xd800) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); - JUMPHERE(jump); - } -#endif -} - -static SLJIT_INLINE void read_char(compiler_common *common) -{ -read_char_range(common, 0, READ_CHAR_MAX, TRUE); -} - -static void read_char8_type(compiler_common *common, BOOL update_str_ptr) +static void read_char8_type(compiler_common *common, jump_list **backtracks, BOOL negated) { /* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */ DEFINE_COMPILER; @@ -3391,7 +3755,8 @@ struct sljit_jump *jump; struct sljit_jump *jump2; #endif -SLJIT_UNUSED_ARG(update_str_ptr); +SLJIT_UNUSED_ARG(backtracks); +SLJIT_UNUSED_ARG(negated); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -3399,18 +3764,38 @@ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf) { - /* This can be an extra read in some situations, but hopefully - it is needed in most cases. */ + /* The result of this read may be unused, but saves an "else" part. */ OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); - jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); - if (!update_str_ptr) + jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x80); + + if (!negated) { + if (common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2); + if (common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe0 - 0xc2)); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80); + if (common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40)); + + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); + JUMPHERE(jump2); + } + else if (common->invalid_utf) + { + add_jump(compiler, &common->utfreadchar_invalid, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, TMP2, 0, TMP1, 0); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); @@ -3418,43 +3803,98 @@ if (common->utf) } else add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL)); + JUMPHERE(jump); return; } #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 */ +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 32 +if (common->invalid_utf && negated) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x110000)); +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 32 */ + #if PCRE2_CODE_UNIT_WIDTH != 8 /* The ctypes array contains only 256 values. */ OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */ OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); #if PCRE2_CODE_UNIT_WIDTH != 8 JUMPHERE(jump); -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH != 8 */ #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16 -if (common->utf && update_str_ptr) +if (common->utf && negated) { /* Skip low surrogate if necessary. */ + if (!common->invalid_utf) + { + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); + + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && sljit_get_register_index(RETURN_ADDR) >= 0) + { + OP2(SLJIT_ADD, RETURN_ADDR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x400); + CMOV(SLJIT_LESS, STR_PTR, RETURN_ADDR, 0); + } + else + { + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPHERE(jump); + } + return; + } + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); + jump = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0xe000 - 0xd800); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400)); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xdc00); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400)); + JUMPHERE(jump); + return; } #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 16 */ } -static void skip_char_back(compiler_common *common) +static void move_back(compiler_common *common, jump_list **backtracks, BOOL must_be_valid) { -/* Goes one character back. Affects STR_PTR and TMP1. Does not check begin. */ +/* Goes one character back. TMP2 must contain the start of +the subject buffer. Affects STR_PTR and TMP1. Does not modify +STR_PTR for invalid character sequences. */ DEFINE_COMPILER; + +SLJIT_UNUSED_ARG(backtracks); +SLJIT_UNUSED_ARG(must_be_valid); + #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +struct sljit_jump *jump; +#endif + +#ifdef SUPPORT_UNICODE #if PCRE2_CODE_UNIT_WIDTH == 8 struct sljit_label *label; if (common->utf) { + if (!must_be_valid && common->invalid_utf) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x80); + add_jump(compiler, &common->utfmoveback_invalid, JUMP(SLJIT_FAST_CALL)); + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0)); + JUMPHERE(jump); + return; + } + label = LABEL(); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); @@ -3467,16 +3907,45 @@ if (common->utf) { OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + + if (!must_be_valid && common->invalid_utf) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xe000 - 0xd800); + add_jump(compiler, &common->utfmoveback_invalid, JUMP(SLJIT_FAST_CALL)); + if (backtracks != NULL) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0)); + JUMPHERE(jump); + return; + } + /* Skip low surrogate if necessary. */ OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); return; } -#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */ -#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ +#elif PCRE2_CODE_UNIT_WIDTH == 32 +if (common->invalid_utf && !must_be_valid) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); + if (backtracks != NULL) + { + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x110000)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + return; + } + + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x110000); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_LESS); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + return; + } +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16|32] */ +#endif /* SUPPORT_UNICODE */ OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); } @@ -3519,13 +3988,12 @@ else static void do_utfreadchar(compiler_common *common) { /* Fast decoding a UTF-8 character. TMP1 contains the first byte -of the character (>= 0xc0). Return char value in TMP1, length in TMP2. */ +of the character (>= 0xc0). Return char value in TMP1. */ DEFINE_COMPILER; struct sljit_jump *jump; sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); @@ -3534,13 +4002,12 @@ OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); jump = JUMP(SLJIT_NOT_ZERO); /* Two byte sequence. */ +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3000); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(2)); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); @@ -3548,55 +4015,18 @@ OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); jump = JUMP(SLJIT_NOT_ZERO); /* Three byte sequence. */ +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0000); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(3)); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); /* Four byte sequence. */ JUMPHERE(jump); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xf0000); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(4)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -static void do_utfreadchar16(compiler_common *common) -{ -/* Fast decoding a UTF-8 character. TMP1 contains the first byte -of the character (>= 0xc0). Return value in TMP1. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); -jump = JUMP(SLJIT_NOT_ZERO); -/* Two byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -JUMPHERE(jump); -OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_NOT_ZERO); -/* This code runs only in 8 bit mode. No need to shift the value. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -/* Three byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -3636,8 +4066,644 @@ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } +static void do_utfreadchar_invalid(compiler_common *common) +{ +/* Slow decoding a UTF-8 character. TMP1 contains the first byte +of the character (>= 0xc0). Return char value in TMP1. STR_PTR is +undefined for invalid characters. */ +DEFINE_COMPILER; +sljit_s32 i; +sljit_s32 has_cmov = sljit_has_cpu_feature(SLJIT_HAS_CMOV); +struct sljit_jump *jump; +struct sljit_jump *buffer_end_close; +struct sljit_label *three_byte_entry; +struct sljit_label *exit_invalid_label; +struct sljit_jump *exit_invalid[11]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc2); + +/* Usually more than 3 characters remained in the subject buffer. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); + +/* Not a valid start of a multi-byte sequence, no more bytes read. */ +exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xf5 - 0xc2); + +buffer_end_close = CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-3)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +/* If TMP2 is in 0x80-0xbf range, TMP1 is also increased by (0x2 << 6). */ +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); + +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +jump = JUMP(SLJIT_NOT_ZERO); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump); + +/* Three-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x40); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, 0x20000); + exit_invalid[2] = NULL; + } +else + exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); + +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); +jump = JUMP(SLJIT_NOT_ZERO); + +three_byte_entry = LABEL(); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2d800); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, INVALID_UTF_CHAR - 0xd800); + exit_invalid[3] = NULL; + } +else + exit_invalid[3] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + exit_invalid[4] = NULL; + } +else + exit_invalid[4] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump); + +/* Four-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x40); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, 0); + exit_invalid[5] = NULL; + } +else + exit_invalid[5] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc10000); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x100000); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR - 0x10000); + exit_invalid[6] = NULL; + } +else + exit_invalid[6] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x100000); + +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(buffer_end_close); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); +exit_invalid[7] = CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0); + +/* Two-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +/* If TMP2 is in 0x80-0xbf range, TMP1 is also increased by (0x2 << 6). */ +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +exit_invalid[8] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); + +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +jump = JUMP(SLJIT_NOT_ZERO); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Three-byte sequence. */ +JUMPHERE(jump); +exit_invalid[9] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x40); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + exit_invalid[10] = NULL; + } +else + exit_invalid[10] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); + +/* One will be substracted from STR_PTR later. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + +/* Four byte sequences are not possible. */ +CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x30000, three_byte_entry); + +exit_invalid_label = LABEL(); +for (i = 0; i < 11; i++) + sljit_set_label(exit_invalid[i], exit_invalid_label); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfreadnewline_invalid(compiler_common *common) +{ +/* Slow decoding a UTF-8 character, specialized for newlines. +TMP1 contains the first byte of the character (>= 0xc0). Return +char value in TMP1. */ +DEFINE_COMPILER; +struct sljit_label *loop; +struct sljit_label *skip_start; +struct sljit_label *three_byte_exit; +struct sljit_jump *jump[5]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +if (common->nltype != NLTYPE_ANY) + { + SLJIT_ASSERT(common->nltype != NLTYPE_FIXED || common->newline < 128); + + /* All newlines are ascii, just skip intermediate octets. */ + jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + loop = LABEL(); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc0); + CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0x80, loop); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + + JUMPHERE(jump[0]); + + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + return; + } + +jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +jump[1] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xc2); +jump[2] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xe2); + +skip_start = LABEL(); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc0); +jump[3] = CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0x80); + +/* Skip intermediate octets. */ +loop = LABEL(); +jump[4] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc0); +CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0x80, loop); + +JUMPHERE(jump[3]); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +three_byte_exit = LABEL(); +JUMPHERE(jump[0]); +JUMPHERE(jump[4]); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Two byte long newline: 0x85. */ +JUMPHERE(jump[1]); +CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0x85, skip_start); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x85); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Three byte long newlines: 0x2028 and 0x2029. */ +JUMPHERE(jump[2]); +CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0x80, skip_start); +CMPTO(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0, three_byte_exit); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +OP2(SLJIT_SUB, TMP1, 0, TMP2, 0, SLJIT_IMM, 0x80); +CMPTO(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x40, skip_start); + +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0x2000); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfmoveback_invalid(compiler_common *common) +{ +/* Goes one character back. */ +DEFINE_COMPILER; +sljit_s32 i; +struct sljit_jump *jump; +struct sljit_jump *buffer_start_close; +struct sljit_label *exit_ok_label; +struct sljit_label *exit_invalid_label; +struct sljit_jump *exit_invalid[7]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); +exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xc0); + +/* Two-byte sequence. */ +buffer_start_close = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); +jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x20); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Three-byte sequence. */ +JUMPHERE(jump); +exit_invalid[1] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, -0x40); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0); +jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x10); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Four-byte sequence. */ +JUMPHERE(jump); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0 - 0x80); +exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x40); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xf0); +exit_invalid[3] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x05); + +exit_ok_label = LABEL(); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Two-byte sequence. */ +JUMPHERE(buffer_start_close); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + +exit_invalid[4] = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); +CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x20, exit_ok_label); + +/* Three-byte sequence. */ +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +exit_invalid[5] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, -0x40); +exit_invalid[6] = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0); +CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10, exit_ok_label); + +/* Four-byte sequences are not possible. */ + +exit_invalid_label = LABEL(); +sljit_set_label(exit_invalid[5], exit_invalid_label); +sljit_set_label(exit_invalid[6], exit_invalid_label); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(exit_invalid[4]); +/* -2 + 4 = 2 */ +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + +exit_invalid_label = LABEL(); +for (i = 0; i < 4; i++) + sljit_set_label(exit_invalid[i], exit_invalid_label); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(4)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfpeakcharback(compiler_common *common) +{ +/* Peak a character back. */ +DEFINE_COMPILER; +struct sljit_jump *jump[2]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); +jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x20); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-3)); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0); +jump[1] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-4)); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0 - 0x80); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf0); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +JUMPHERE(jump[1]); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +JUMPHERE(jump[0]); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x80); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfpeakcharback_invalid(compiler_common *common) +{ +/* Peak a character back. */ +DEFINE_COMPILER; +sljit_s32 i; +sljit_s32 has_cmov = sljit_has_cpu_feature(SLJIT_HAS_CMOV); +struct sljit_jump *jump[2]; +struct sljit_label *two_byte_entry; +struct sljit_label *three_byte_entry; +struct sljit_label *exit_invalid_label; +struct sljit_jump *exit_invalid[8]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(3)); +exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xc0); +jump[0] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, STR_PTR, 0); + +/* Two-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2); +jump[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x1e); + +two_byte_entry = LABEL(); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); +/* If TMP1 is in 0x80-0xbf range, TMP1 is also increased by (0x2 << 6). */ +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump[1]); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2 - 0x80); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* Three-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-3)); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xe0); +jump[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x10); + +three_byte_entry = LABEL(); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, -0xd800); + exit_invalid[2] = NULL; + } +else + exit_invalid[2] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); + +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); + CMOV(SLJIT_LESS, TMP1, SLJIT_IMM, INVALID_UTF_CHAR); + exit_invalid[3] = NULL; + } +else + exit_invalid[3] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x800); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump[1]); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xe0 - 0x80); +exit_invalid[4] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 12); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* Four-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-4)); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf0); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 18); +/* ADD is used instead of OR because of the SUB 0x10000 above. */ +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + +if (has_cmov) + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x100000); + CMOV(SLJIT_GREATER_EQUAL, TMP1, SLJIT_IMM, INVALID_UTF_CHAR - 0x10000); + exit_invalid[5] = NULL; + } +else + exit_invalid[5] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x100000); + +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump[0]); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +jump[0] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, STR_PTR, 0); + +/* Two-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2); +CMPTO(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x1e, two_byte_entry); + +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2 - 0x80); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80); +exit_invalid[6] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x40); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* Three-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-3)); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xe0); +CMPTO(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x10, three_byte_entry); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump[0]); +exit_invalid[7] = CMP(SLJIT_GREATER, TMP2, 0, STR_PTR, 0); + +/* Two-byte sequence. */ +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xc2); +CMPTO(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0x1e, two_byte_entry); + +exit_invalid_label = LABEL(); +for (i = 0; i < 8; i++) + sljit_set_label(exit_invalid[i], exit_invalid_label); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + #endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ +#if PCRE2_CODE_UNIT_WIDTH == 16 + +static void do_utfreadchar_invalid(compiler_common *common) +{ +/* Slow decoding a UTF-16 character. TMP1 contains the first half +of the character (>= 0xd800). Return char value in TMP1. STR_PTR is +undefined for invalid characters. */ +DEFINE_COMPILER; +struct sljit_jump *exit_invalid[3]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +/* TMP2 contains the high surrogate. */ +exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xdc00); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x10000); +exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x400); + +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(exit_invalid[0]); +JUMPHERE(exit_invalid[1]); +JUMPHERE(exit_invalid[2]); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfreadnewline_invalid(compiler_common *common) +{ +/* Slow decoding a UTF-16 character, specialized for newlines. +TMP1 contains the first half of the character (>= 0xd800). Return +char value in TMP1. */ + +DEFINE_COMPILER; +struct sljit_jump *exit_invalid[2]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +/* TMP2 contains the high surrogate. */ +exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00); + +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xdc00); +OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x400); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(exit_invalid[0]); +JUMPHERE(exit_invalid[1]); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfmoveback_invalid(compiler_common *common) +{ +/* Goes one character back. */ +DEFINE_COMPILER; +struct sljit_jump *exit_invalid[3]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +exit_invalid[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x400); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, STR_PTR, 0); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); +exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x400); + +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(exit_invalid[0]); +JUMPHERE(exit_invalid[1]); +JUMPHERE(exit_invalid[2]); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfpeakcharback_invalid(compiler_common *common) +{ +/* Peak a character back. */ +DEFINE_COMPILER; +struct sljit_jump *jump; +struct sljit_jump *exit_invalid[3]; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xe000); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +exit_invalid[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xdc00); +exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, STR_PTR, 0); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000 - 0xdc00); +OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); +exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP2, 0, SLJIT_IMM, 0x400); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + +JUMPHERE(jump); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(exit_invalid[0]); +JUMPHERE(exit_invalid[1]); +JUMPHERE(exit_invalid[2]); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +#endif /* PCRE2_CODE_UNIT_WIDTH == 16 */ + /* UCD_BLOCK_SIZE must be 128 (see the assert below). */ #define UCD_BLOCK_MASK 127 #define UCD_BLOCK_SHIFT 7 @@ -3653,12 +4719,12 @@ struct sljit_jump *jump; #if defined SLJIT_DEBUG && SLJIT_DEBUG /* dummy_ucd_record */ -const ucd_record *record = GET_UCD(INVALID_UTF_CHAR); -SLJIT_ASSERT(record->script == ucp_Common && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); +const ucd_record *record = GET_UCD(UNASSIGNED_UTF_CHAR); +SLJIT_ASSERT(record->script == ucp_Unknown && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0); #endif -SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8); +SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 12); sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); @@ -3666,7 +4732,7 @@ sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); if (!common->utf) { jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, MAX_UTF_CODE_POINT + 1); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, UNASSIGNED_UTF_CHAR); JUMPHERE(jump); } #endif @@ -3679,8 +4745,59 @@ OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_getucdtype(compiler_common *common) +{ +/* Search the UCD record for the character comes in TMP1. +Returns chartype in TMP1 and UCD offset in TMP2. */ +DEFINE_COMPILER; +#if PCRE2_CODE_UNIT_WIDTH == 32 +struct sljit_jump *jump; +#endif + +#if defined SLJIT_DEBUG && SLJIT_DEBUG +/* dummy_ucd_record */ +const ucd_record *record = GET_UCD(UNASSIGNED_UTF_CHAR); +SLJIT_ASSERT(record->script == ucp_Unknown && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); +SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0); +#endif + +SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 12); + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +#if PCRE2_CODE_UNIT_WIDTH == 32 +if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, MAX_UTF_CODE_POINT + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, UNASSIGNED_UTF_CHAR); + JUMPHERE(jump); + } +#endif + +OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); +OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); +OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); +OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); + +// PH hacking +//fprintf(stderr, "~~A\n"); + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); + +// OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); sljit_emit_fast_return(compiler, RETURN_ADDR, 0); } @@ -3695,8 +4812,9 @@ struct sljit_jump *start; struct sljit_jump *end = NULL; struct sljit_jump *end2 = NULL; #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 -struct sljit_jump *singlechar; -#endif +struct sljit_label *loop; +struct sljit_jump *jump; +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ jump_list *newline = NULL; sljit_u32 overall_options = common->re->overall_options; BOOL hascrorlf = (common->re->flags & PCRE2_HASCRORLF) != 0; @@ -3733,7 +4851,7 @@ if ((overall_options & PCRE2_FIRSTLINE) != 0) mainloop = LABEL(); /* Continual stores does not cause data dependency. */ OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0); - read_char_range(common, common->nlmin, common->nlmax, TRUE); + read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE); check_newlinechar(common, common->nltype, &newline, TRUE); CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, mainloop); JUMPHERE(end); @@ -3753,11 +4871,9 @@ else if ((overall_options & PCRE2_USE_OFFSET_LIMIT) != 0) OP1(SLJIT_MOV, TMP2, 0, STR_END, 0); end = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw) PCRE2_UNSET); OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); -#if PCRE2_CODE_UNIT_WIDTH == 16 - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); -#elif PCRE2_CODE_UNIT_WIDTH == 32 - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 2); -#endif +#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0); @@ -3781,7 +4897,7 @@ if (newlinecheck) OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); #if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32 OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); end2 = JUMP(SLJIT_JUMP); } @@ -3789,9 +4905,9 @@ if (newlinecheck) mainloop = LABEL(); /* Increasing the STR_PTR here requires one less jump in the most common case. */ -#ifdef SUPPORT_UNICODE -if (common->utf) readuchar = TRUE; -#endif +#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +if (common->utf && !common->invalid_utf) readuchar = TRUE; +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ if (newlinecheck) readuchar = TRUE; if (readuchar) @@ -3803,23 +4919,55 @@ if (newlinecheck) OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 #if PCRE2_CODE_UNIT_WIDTH == 8 -if (common->utf) +if (common->invalid_utf) { - singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + /* Skip continuation code units. */ + loop = LABEL(); + jump = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80); + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x40, loop); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPHERE(jump); + } +else if (common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(singlechar); + JUMPHERE(jump); } #elif PCRE2_CODE_UNIT_WIDTH == 16 -if (common->utf) +if (common->invalid_utf) { - singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(singlechar); + /* Skip continuation code units. */ + loop = LABEL(); + jump = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xdc00); + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x400, loop); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPHERE(jump); + } +else if (common->utf) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800); + + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV)) + { + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); + CMOV(SLJIT_LESS, STR_PTR, TMP2, 0); + } + else + { + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_LESS); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + } } #endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */ #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */ @@ -4305,16 +5453,16 @@ return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0xdc00); static sljit_s32 character_to_int32(PCRE2_UCHAR chr) { -sljit_s32 value = (sljit_s32)chr; +sljit_u32 value = chr; #if PCRE2_CODE_UNIT_WIDTH == 8 #define SSE2_COMPARE_TYPE_INDEX 0 -return (value << 24) | (value << 16) | (value << 8) | value; +return (sljit_s32)((value << 24) | (value << 16) | (value << 8) | value); #elif PCRE2_CODE_UNIT_WIDTH == 16 #define SSE2_COMPARE_TYPE_INDEX 1 -return (value << 16) | value; +return (sljit_s32)((value << 16) | value); #elif PCRE2_CODE_UNIT_WIDTH == 32 #define SSE2_COMPARE_TYPE_INDEX 2 -return value; +return (sljit_s32)(value); #else #error "Unsupported unit width" #endif @@ -5120,7 +6268,7 @@ for (i = 0; i < max; i++) } #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) && !(defined _WIN64) -if (check_fast_forward_char_pair_sse2(common, chars, max)) +if (sljit_has_cpu_feature(SLJIT_HAS_SSE2) && check_fast_forward_char_pair_sse2(common, chars, max)) return TRUE; #endif @@ -5356,14 +6504,15 @@ if (common->nltype == NLTYPE_FIXED && common->newline > 255) } OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +/* Example: match /^/ to \r\n from offset 1. */ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); -skip_char_back(common); +move_back(common, NULL, FALSE); loop = LABEL(); common->ff_newline_shortcut = loop; -read_char_range(common, common->nlmin, common->nlmax, TRUE); +read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE); lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); @@ -5544,7 +6693,7 @@ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw)); jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); -if (sljit_get_register_index (TMP3) < 0) +if (sljit_get_register_index(TMP3) < 0) { OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -(2 * sizeof(sljit_sw))); OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -(3 * sizeof(sljit_sw))); @@ -5569,7 +6718,7 @@ sljit_emit_fast_return(compiler, RETURN_ADDR, 0); JUMPHERE(jump); OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); -if (sljit_get_register_index (TMP3) < 0) +if (sljit_get_register_index(TMP3) < 0) { OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -(2 * sizeof(sljit_sw))); OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); @@ -5588,21 +6737,29 @@ static void check_wordboundary(compiler_common *common) DEFINE_COMPILER; struct sljit_jump *skipread; jump_list *skipread_list = NULL; +jump_list *invalid_utf = NULL; #if PCRE2_CODE_UNIT_WIDTH != 8 || defined SUPPORT_UNICODE struct sljit_jump *jump; -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH != 8 || SUPPORT_UNICODE */ SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16); sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); -/* Get type of the previous char, and put it to LOCALS1. */ +/* Get type of the previous char, and put it to TMP3. */ OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, SLJIT_IMM, 0); -skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0); -skip_char_back(common); -check_start_used_ptr(common); -read_char(common); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); +OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); +skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); + +if (common->mode == PCRE2_JIT_COMPLETE) + peek_char_back(common, READ_CHAR_MAX, &invalid_utf); +else + { + move_back(common, &invalid_utf, FALSE); + check_start_used_ptr(common); + /* No need precise read since match fails anyway. */ + read_char(common, 0, READ_CHAR_MAX, &invalid_utf, READ_CHAR_UPDATE_STR_PTR); + } /* Testing char type. */ #ifdef SUPPORT_UNICODE @@ -5610,7 +6767,7 @@ if (common->use_ucp) { OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + add_jump(compiler, &common->getucdtype, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); @@ -5618,23 +6775,22 @@ if (common->use_ucp) OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); JUMPHERE(jump); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); + OP1(SLJIT_MOV, TMP3, 0, TMP2, 0); } else -#endif +#endif /* SUPPORT_UNICODE */ { #if PCRE2_CODE_UNIT_WIDTH != 8 jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); #elif defined SUPPORT_UNICODE - /* Here LOCALS1 has already been zeroed. */ + /* Here TMP3 has already been zeroed. */ jump = NULL; if (common->utf) jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); #endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes); OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); + OP2(SLJIT_AND, TMP3, 0, TMP1, 0, SLJIT_IMM, 1); #if PCRE2_CODE_UNIT_WIDTH != 8 JUMPHERE(jump); #elif defined SUPPORT_UNICODE @@ -5646,7 +6802,7 @@ JUMPHERE(skipread); OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); check_str_end(common, &skipread_list); -peek_char(common, READ_CHAR_MAX); +peek_char(common, READ_CHAR_MAX, SLJIT_MEM1(SLJIT_SP), LOCALS1, &invalid_utf); /* Testing char type. This is a code duplication. */ #ifdef SUPPORT_UNICODE @@ -5654,7 +6810,7 @@ if (common->use_ucp) { OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + add_jump(compiler, &common->getucdtype, JUMP(SLJIT_FAST_CALL)); OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); @@ -5664,7 +6820,7 @@ if (common->use_ucp) JUMPHERE(jump); } else -#endif +#endif /* SUPPORT_UNICODE */ { #if PCRE2_CODE_UNIT_WIDTH != 8 /* TMP2 may be destroyed by peek_char. */ @@ -5688,8 +6844,22 @@ else } set_jumps(skipread_list, LABEL()); -OP2(SLJIT_XOR | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); +OP2(SLJIT_XOR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, TMP3, 0); +sljit_emit_fast_return(compiler, TMP1, 0); + +#ifdef SUPPORT_UNICODE +if (common->invalid_utf) + { + SLJIT_ASSERT(invalid_utf != NULL); + + set_jumps(invalid_utf, LABEL()); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, -1); + sljit_emit_fast_return(compiler, TMP1, 0); + return; + } +#endif /* SUPPORT_UNICODE */ } static BOOL optimize_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks) @@ -5856,9 +7026,6 @@ int i, j, k, len, c; if (!sljit_has_cpu_feature(SLJIT_HAS_CMOV)) return FALSE; -if (invert) - nclass = !nclass; - len = 0; for (i = 0; i < 32; i++) @@ -5940,6 +7107,9 @@ if (j != 0) } } +if (invert) + nclass = !nclass; + type = nclass ? SLJIT_NOT_EQUAL : SLJIT_EQUAL; add_jump(compiler, backtracks, CMP(type, TMP2, 0, SLJIT_IMM, 0)); return TRUE; @@ -6225,37 +7395,6 @@ OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); sljit_emit_fast_return(compiler, TMP1, 0); } -#if defined SUPPORT_UNICODE - -static PCRE2_SPTR SLJIT_FUNC do_utf_caselesscmp(PCRE2_SPTR src1, PCRE2_SPTR src2, PCRE2_SPTR end1, PCRE2_SPTR end2) -{ -/* This function would be ineffective to do in JIT level. */ -sljit_u32 c1, c2; -const ucd_record *ur; -const sljit_u32 *pp; - -while (src1 < end1) - { - if (src2 >= end2) - return (PCRE2_SPTR)1; - GETCHARINC(c1, src1); - GETCHARINC(c2, src2); - ur = GET_UCD(c2); - if (c1 != c2 && c1 != c2 + ur->other_case) - { - pp = PRIV(ucd_caseless_sets) + ur->caseset; - for (;;) - { - if (c1 < *pp) return NULL; - if (c1 == *pp++) break; - } - } - } -return src2; -} - -#endif /* SUPPORT_UNICODE */ - static PCRE2_SPTR byte_sequence_compare(compiler_common *common, BOOL caseless, PCRE2_SPTR cc, compare_context *context, jump_list **backtracks) { @@ -6297,7 +7436,7 @@ if (context->sourcereg == -1) OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); else #endif - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); #elif PCRE2_CODE_UNIT_WIDTH == 16 #if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED if (context->length >= 4) @@ -6444,7 +7583,7 @@ PCRE2_SPTR ccbegin; int compares, invertcmp, numberofcmps; #if defined SUPPORT_UNICODE && (PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16) BOOL utf = common->utf; -#endif +#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == [8|16] */ #ifdef SUPPORT_UNICODE BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE; @@ -6452,7 +7591,7 @@ BOOL charsaved = FALSE; int typereg = TMP1; const sljit_u32 *other_cases; sljit_uw typeoffset; -#endif +#endif /* SUPPORT_UNICODE */ /* Scanning the necessary info. */ cc++; @@ -6476,7 +7615,7 @@ while (*cc != XCL_END) if (c < min) min = c; #ifdef SUPPORT_UNICODE needschar = TRUE; -#endif +#endif /* SUPPORT_UNICODE */ } else if (*cc == XCL_RANGE) { @@ -6487,7 +7626,7 @@ while (*cc != XCL_END) if (c > max) max = c; #ifdef SUPPORT_UNICODE needschar = TRUE; -#endif +#endif /* SUPPORT_UNICODE */ } #ifdef SUPPORT_UNICODE else @@ -6555,13 +7694,16 @@ while (*cc != XCL_END) } cc += 2; } -#endif +#endif /* SUPPORT_UNICODE */ } SLJIT_ASSERT(compares > 0); /* We are not necessary in utf mode even in 8 bit mode. */ cc = ccbegin; -read_char_range(common, min, max, (cc[-1] & XCL_NOT) != 0); +if ((cc[-1] & XCL_NOT) != 0) + read_char(common, min, max, backtracks, READ_CHAR_UPDATE_STR_PTR); +else + read_char(common, min, max, NULL, 0); if ((cc[-1] & XCL_HASPROP) == 0) { @@ -6594,13 +7736,13 @@ else if ((cc[-1] & XCL_MAP) != 0) OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); #ifdef SUPPORT_UNICODE charsaved = TRUE; -#endif +#endif /* SUPPORT_UNICODE */ if (!optimize_class(common, (const sljit_u8 *)cc, FALSE, TRUE, list)) { #if PCRE2_CODE_UNIT_WIDTH == 8 jump = NULL; if (common->utf) -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); @@ -6612,7 +7754,7 @@ else if ((cc[-1] & XCL_MAP) != 0) #if PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf) -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ JUMPHERE(jump); } @@ -6630,10 +7772,10 @@ if (needstype || needsscript) if (!common->utf) { jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, MAX_UTF_CODE_POINT + 1); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, UNASSIGNED_UTF_CHAR); JUMPHERE(jump); } -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH == 32 */ OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); @@ -6647,8 +7789,18 @@ if (needstype || needsscript) /* Before anything else, we deal with scripts. */ if (needsscript) { +// PH hacking +//fprintf(stderr, "~~B\n"); + + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script)); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); + + // OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); ccbegin = cc; @@ -6686,33 +7838,49 @@ if (needstype || needsscript) } if (needschar) - { OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); - } if (needstype) { if (!needschar) { +// PH hacking +//fprintf(stderr, "~~C\n"); + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP2(SLJIT_ADD, TMP1, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 0); + +// OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); } else { +// PH hacking +//fprintf(stderr, "~~D\n"); + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); typereg = RETURN_ADDR; } } } -#endif +#endif /* SUPPORT_UNICODE */ /* Generating code. */ charoffset = 0; numberofcmps = 0; #ifdef SUPPORT_UNICODE typeoffset = 0; -#endif +#endif /* SUPPORT_UNICODE */ while (*cc != XCL_END) { @@ -6979,7 +8147,7 @@ while (*cc != XCL_END) } cc += 2; } -#endif +#endif /* SUPPORT_UNICODE */ if (jump != NULL) add_jump(compiler, compares > 0 ? list : backtracks, jump); @@ -7020,6 +8188,15 @@ switch(type) case OP_NOT_WORD_BOUNDARY: case OP_WORD_BOUNDARY: add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL)); +#ifdef SUPPORT_UNICODE + if (common->invalid_utf) + { + OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_LESS, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0); + add_jump(compiler, backtracks, JUMP(SLJIT_SIG_LESS)); + add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + return cc; + } +#endif /* SUPPORT_UNICODE */ sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); return cc; @@ -7078,13 +8255,13 @@ switch(type) } else { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, STR_PTR, 0); - read_char_range(common, common->nlmin, common->nlmax, TRUE); + OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + read_char(common, common->nlmin, common->nlmax, backtracks, READ_CHAR_UPDATE_STR_PTR); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0)); add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); } JUMPHERE(jump[2]); JUMPHERE(jump[3]); @@ -7143,7 +8320,7 @@ switch(type) } else { - peek_char(common, common->nlmax); + peek_char(common, common->nlmax, TMP3, 0, NULL); check_newlinechar(common, common->nltype, backtracks, FALSE); } JUMPHERE(jump[0]); @@ -7158,10 +8335,10 @@ switch(type) return cc; case OP_CIRCM: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); - jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0); - OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP2, 0); + OP2(SLJIT_AND32 | SLJIT_SET_Z, SLJIT_UNUSED, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, options), SLJIT_IMM, PCRE2_NOTBOL); add_jump(compiler, backtracks, JUMP(SLJIT_NOT_ZERO32)); jump[0] = JUMP(SLJIT_JUMP); JUMPHERE(jump[1]); @@ -7171,8 +8348,8 @@ switch(type) if (common->nltype == NLTYPE_FIXED && common->newline > 255) { - OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, TMP1, 0)); + OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, TMP2, 0)); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); @@ -7180,8 +8357,7 @@ switch(type) } else { - skip_char_back(common); - read_char_range(common, common->nlmin, common->nlmax, TRUE); + peek_char_back(common, common->nlmax, backtracks); check_newlinechar(common, common->nltype, backtracks, FALSE); } JUMPHERE(jump[0]); @@ -7195,12 +8371,12 @@ switch(type) #ifdef SUPPORT_UNICODE if (common->utf) { - OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, length); label = LABEL(); - add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0)); - skip_char_back(common); - OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); + add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0)); + move_back(common, backtracks, FALSE); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP3, 0, TMP3, 0, SLJIT_IMM, 1); JUMPTO(SLJIT_NOT_ZERO, label); } else @@ -7225,21 +8401,28 @@ static PCRE2_SPTR SLJIT_FUNC do_extuni_utf(jit_arguments *args, PCRE2_SPTR cc) { PCRE2_SPTR start_subject = args->begin; PCRE2_SPTR end_subject = args->end; -int lgb, rgb, len, ricount; -PCRE2_SPTR prevcc, bptr; +int lgb, rgb, ricount; +PCRE2_SPTR prevcc, startcc, bptr; +BOOL first = TRUE; uint32_t c; prevcc = cc; -GETCHARINC(c, cc); -lgb = UCD_GRAPHBREAK(c); - -while (cc < end_subject) +startcc = NULL; +do { - len = 1; - GETCHARLEN(c, cc, len); + GETCHARINC(c, cc); rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + if (first) + { + lgb = rgb; + startcc = cc; + first = FALSE; + continue; + } + + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) + break; /* Not breaking between Regional Indicators is allowed only if there are an even number of preceding RIs. */ @@ -7256,7 +8439,8 @@ while (cc < end_subject) BACKCHAR(bptr); GETCHAR(c, bptr); - if (UCD_GRAPHBREAK(c) != ucp_gbRegionalIndicator) break; + if (UCD_GRAPHBREAK(c) != ucp_gbRegionalIndicator) + break; ricount++; } @@ -7271,14 +8455,80 @@ while (cc < end_subject) lgb != ucp_gbExtended_Pictographic) lgb = rgb; - prevcc = cc; - cc += len; + prevcc = startcc; + startcc = cc; } +while (cc < end_subject); -return cc; +return startcc; } -#endif +static PCRE2_SPTR SLJIT_FUNC do_extuni_utf_invalid(jit_arguments *args, PCRE2_SPTR cc) +{ +PCRE2_SPTR start_subject = args->begin; +PCRE2_SPTR end_subject = args->end; +int lgb, rgb, ricount; +PCRE2_SPTR prevcc, startcc, bptr; +BOOL first = TRUE; +uint32_t c; + +prevcc = cc; +startcc = NULL; +do + { + GETCHARINC_INVALID(c, cc, end_subject, break); + rgb = UCD_GRAPHBREAK(c); + + if (first) + { + lgb = rgb; + startcc = cc; + first = FALSE; + continue; + } + + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) + break; + + /* Not breaking between Regional Indicators is allowed only if there + are an even number of preceding RIs. */ + + if (lgb == ucp_gbRegionalIndicator && rgb == ucp_gbRegionalIndicator) + { + ricount = 0; + bptr = prevcc; + + /* bptr is pointing to the left-hand character */ + while (bptr > start_subject) + { + GETCHARBACK_INVALID(c, bptr, start_subject, break); + + if (UCD_GRAPHBREAK(c) != ucp_gbRegionalIndicator) + break; + + ricount++; + } + + if ((ricount & 1) != 0) + break; /* Grapheme break required */ + } + + /* If Extend or ZWJ follows Extended_Pictographic, do not update lgb; this + allows any number of them before a following Extended_Pictographic. */ + + if ((rgb != ucp_gbExtend && rgb != ucp_gbZWJ) || + lgb != ucp_gbExtended_Pictographic) + lgb = rgb; + + prevcc = startcc; + startcc = cc; + } +while (cc < end_subject); + +return startcc; +} + +#endif /* PCRE2_CODE_UNIT_WIDTH != 32 */ static PCRE2_SPTR SLJIT_FUNC do_extuni_no_utf(jit_arguments *args, PCRE2_SPTR cc) { @@ -7289,14 +8539,23 @@ PCRE2_SPTR bptr; uint32_t c; GETCHARINC(c, cc); +#if PCRE2_CODE_UNIT_WIDTH == 32 +if (c >= 0x110000) + return NULL; +#endif /* PCRE2_CODE_UNIT_WIDTH == 32 */ lgb = UCD_GRAPHBREAK(c); while (cc < end_subject) { c = *cc; +#if PCRE2_CODE_UNIT_WIDTH == 32 + if (c >= 0x110000) + break; +#endif /* PCRE2_CODE_UNIT_WIDTH == 32 */ rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) + break; /* Not breaking between Regional Indicators is allowed only if there are an even number of preceding RIs. */ @@ -7311,13 +8570,18 @@ while (cc < end_subject) { bptr--; c = *bptr; +#if PCRE2_CODE_UNIT_WIDTH == 32 + if (c >= 0x110000) + break; +#endif /* PCRE2_CODE_UNIT_WIDTH == 32 */ if (UCD_GRAPHBREAK(c) != ucp_gbRegionalIndicator) break; ricount++; } - if ((ricount & 1) != 0) break; /* Grapheme break required */ + if ((ricount & 1) != 0) + break; /* Grapheme break required */ } /* If Extend or ZWJ follows Extended_Pictographic, do not update lgb; this @@ -7333,7 +8597,7 @@ while (cc < end_subject) return cc; } -#endif +#endif /* SUPPORT_UNICODE */ static PCRE2_SPTR compile_char1_matchingpath(compiler_common *common, PCRE2_UCHAR type, PCRE2_SPTR cc, jump_list **backtracks, BOOL check_str_ptr) { @@ -7356,10 +8620,10 @@ switch(type) detect_partial_match(common, backtracks); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_digit, FALSE)) - read_char7_type(common, type == OP_NOT_DIGIT); + read_char7_type(common, backtracks, type == OP_NOT_DIGIT); else #endif - read_char8_type(common, type == OP_NOT_DIGIT); + read_char8_type(common, backtracks, type == OP_NOT_DIGIT); /* Flip the starting bit in the negative case. */ OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO)); @@ -7371,10 +8635,10 @@ switch(type) detect_partial_match(common, backtracks); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_space, FALSE)) - read_char7_type(common, type == OP_NOT_WHITESPACE); + read_char7_type(common, backtracks, type == OP_NOT_WHITESPACE); else #endif - read_char8_type(common, type == OP_NOT_WHITESPACE); + read_char8_type(common, backtracks, type == OP_NOT_WHITESPACE); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -7385,10 +8649,10 @@ switch(type) detect_partial_match(common, backtracks); #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_word, FALSE)) - read_char7_type(common, type == OP_NOT_WORDCHAR); + read_char7_type(common, backtracks, type == OP_NOT_WORDCHAR); else #endif - read_char8_type(common, type == OP_NOT_WORDCHAR); + read_char8_type(common, backtracks, type == OP_NOT_WORDCHAR); OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO)); return cc; @@ -7396,7 +8660,7 @@ switch(type) case OP_ANY: if (check_str_ptr) detect_partial_match(common, backtracks); - read_char_range(common, common->nlmin, common->nlmax, TRUE); + read_char(common, common->nlmin, common->nlmax, backtracks, READ_CHAR_UPDATE_STR_PTR); if (common->nltype == NLTYPE_FIXED && common->newline > 255) { jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); @@ -7418,12 +8682,18 @@ switch(type) case OP_ALLANY: if (check_str_ptr) detect_partial_match(common, backtracks); -#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 +#ifdef SUPPORT_UNICODE if (common->utf) { + if (common->invalid_utf) + { + read_char(common, 0, READ_CHAR_MAX, backtracks, READ_CHAR_UPDATE_STR_PTR); + return cc; + } + +#if PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16 OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#if PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16 #if PCRE2_CODE_UNIT_WIDTH == 8 jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); @@ -7435,12 +8705,12 @@ switch(type) OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); -#endif +#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ JUMPHERE(jump[0]); -#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */ return cc; +#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */ } -#endif +#endif /* SUPPORT_UNICODE */ OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); return cc; @@ -7467,7 +8737,7 @@ switch(type) case OP_ANYNL: if (check_str_ptr) detect_partial_match(common, backtracks); - read_char_range(common, common->bsr_nlmin, common->bsr_nlmax, FALSE); + read_char(common, common->bsr_nlmin, common->bsr_nlmax, NULL, 0); jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); /* We don't need to handle soft partial matching case. */ end_list = NULL; @@ -7490,7 +8760,12 @@ switch(type) case OP_HSPACE: if (check_str_ptr) detect_partial_match(common, backtracks); - read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE); + + if (type == OP_NOT_HSPACE) + read_char(common, 0x9, 0x3000, backtracks, READ_CHAR_UPDATE_STR_PTR); + else + read_char(common, 0x9, 0x3000, NULL, 0); + add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL)); sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); @@ -7500,7 +8775,12 @@ switch(type) case OP_VSPACE: if (check_str_ptr) detect_partial_match(common, backtracks); - read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE); + + if (type == OP_NOT_VSPACE) + read_char(common, 0xa, 0x2029, backtracks, READ_CHAR_UPDATE_STR_PTR); + else + read_char(common, 0xa, 0x2029, NULL, 0); + add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL)); sljit_set_current_flags(compiler, SLJIT_SET_Z); add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); @@ -7516,9 +8796,12 @@ switch(type) #if PCRE2_CODE_UNIT_WIDTH != 32 sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, - common->utf ? SLJIT_FUNC_OFFSET(do_extuni_utf) : SLJIT_FUNC_OFFSET(do_extuni_no_utf)); + common->utf ? (common->invalid_utf ? SLJIT_FUNC_OFFSET(do_extuni_utf_invalid) : SLJIT_FUNC_OFFSET(do_extuni_utf)) : SLJIT_FUNC_OFFSET(do_extuni_no_utf)); + if (common->invalid_utf) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); #else sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_extuni_no_utf)); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); #endif OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); @@ -7539,11 +8822,15 @@ switch(type) #ifdef SUPPORT_UNICODE if (common->utf && HAS_EXTRALEN(*cc)) length += GET_EXTRALEN(*cc); #endif - if (common->mode == PCRE2_JIT_COMPLETE && check_str_ptr - && (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)) + + if (check_str_ptr && common->mode != PCRE2_JIT_COMPLETE) + detect_partial_match(common, backtracks); + + if (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0) { OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); + if (length > 1 || (check_str_ptr && common->mode == PCRE2_JIT_COMPLETE)) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); context.length = IN_UCHARS(length); context.sourcereg = -1; @@ -7553,8 +8840,6 @@ switch(type) return byte_sequence_compare(common, type == OP_CHARI, cc, &context, backtracks); } - if (check_str_ptr) - detect_partial_match(common, backtracks); #ifdef SUPPORT_UNICODE if (common->utf) { @@ -7564,24 +8849,28 @@ switch(type) #endif c = *cc; - if (type == OP_CHAR || !char_has_othercase(common, cc)) - { - read_char_range(common, c, c, FALSE); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c)); - return cc + length; - } + SLJIT_ASSERT(type == OP_CHARI && char_has_othercase(common, cc)); + + if (check_str_ptr && common->mode == PCRE2_JIT_COMPLETE) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + oc = char_othercase(common, c); - read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, FALSE); - bit = c ^ oc; - if (is_powerof2(bit)) + read_char(common, c < oc ? c : oc, c > oc ? c : oc, NULL, 0); + + SLJIT_ASSERT(!is_powerof2(c ^ oc)); + + if (sljit_has_cpu_feature(SLJIT_HAS_CMOV)) { - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit)); - return cc + length; + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, oc); + CMOV(SLJIT_EQUAL, TMP1, SLJIT_IMM, c); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c)); + } + else + { + jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); + JUMPHERE(jump[0]); } - jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); - JUMPHERE(jump[0]); return cc + length; case OP_NOT: @@ -7595,7 +8884,7 @@ switch(type) { #if PCRE2_CODE_UNIT_WIDTH == 8 c = *cc; - if (c < 128) + if (c < 128 && !common->invalid_utf) { OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); if (type == OP_NOT || !char_has_othercase(common, cc)) @@ -7626,13 +8915,13 @@ switch(type) if (type == OP_NOT || !char_has_othercase(common, cc)) { - read_char_range(common, c, c, TRUE); + read_char(common, c, c, backtracks, READ_CHAR_UPDATE_STR_PTR); add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); } else { oc = char_othercase(common, c); - read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, TRUE); + read_char(common, c < oc ? c : oc, c > oc ? c : oc, backtracks, READ_CHAR_UPDATE_STR_PTR); bit = c ^ oc; if (is_powerof2(bit)) { @@ -7654,9 +8943,15 @@ switch(type) #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 bit = (common->utf && is_char7_bitset((const sljit_u8 *)cc, type == OP_NCLASS)) ? 127 : 255; - read_char_range(common, 0, bit, type == OP_NCLASS); + if (type == OP_NCLASS) + read_char(common, 0, bit, backtracks, READ_CHAR_UPDATE_STR_PTR); + else + read_char(common, 0, bit, NULL, 0); #else - read_char_range(common, 0, 255, type == OP_NCLASS); + if (type == OP_NCLASS) + read_char(common, 0, 255, backtracks, READ_CHAR_UPDATE_STR_PTR); + else + read_char(common, 0, 255, NULL, 0); #endif if (optimize_class(common, (const sljit_u8 *)cc, type == OP_NCLASS, FALSE, backtracks)) @@ -7843,6 +9138,14 @@ int offset = 0; struct sljit_jump *jump = NULL; struct sljit_jump *partial; struct sljit_jump *nopartial; +#if defined SUPPORT_UNICODE +struct sljit_label *loop; +struct sljit_label *caseless_loop; +jump_list *no_match = NULL; +int source_reg = COUNT_MATCH; +int source_end_reg = ARGUMENTS; +int char1_reg = STACK_LIMIT; +#endif /* SUPPORT_UNICODE */ if (ref) { @@ -7858,34 +9161,98 @@ else #if defined SUPPORT_UNICODE if (common->utf && *cc == OP_REFI) { - SLJIT_ASSERT(TMP1 == SLJIT_R0 && STR_PTR == SLJIT_R1); + SLJIT_ASSERT(common->iref_ptr != 0); + if (ref) - OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); else - OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - if (withchecks) - jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_R2, 0); - /* No free saved registers so save data on stack. */ + if (withchecks && emptyfail) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, TMP2, 0)); - OP1(SLJIT_MOV, SLJIT_R3, 0, STR_END, 0); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW) | SLJIT_ARG3(SW) | SLJIT_ARG4(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->iref_ptr, source_reg, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw), source_end_reg, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw) * 2, char1_reg, 0); + OP1(SLJIT_MOV, source_reg, 0, TMP1, 0); + OP1(SLJIT_MOV, source_end_reg, 0, TMP2, 0); + + loop = LABEL(); + jump = CMP(SLJIT_GREATER_EQUAL, source_reg, 0, source_end_reg, 0); + partial = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + + /* Read original character. It must be a valid UTF character. */ + OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + OP1(SLJIT_MOV, STR_PTR, 0, source_reg, 0); + + read_char(common, 0, READ_CHAR_MAX, NULL, READ_CHAR_UPDATE_STR_PTR | READ_CHAR_VALID_UTF); + + OP1(SLJIT_MOV, source_reg, 0, STR_PTR, 0); + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); + OP1(SLJIT_MOV, char1_reg, 0, TMP1, 0); + + /* Read second character. */ + read_char(common, 0, READ_CHAR_MAX, &no_match, READ_CHAR_UPDATE_STR_PTR); + + CMPTO(SLJIT_EQUAL, TMP1, 0, char1_reg, 0, loop); + +// PH hacking +//fprintf(stderr, "~~E\n"); + + OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); + + add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + + OP2(SLJIT_SHL, TMP1, 0, TMP2, 0, SLJIT_IMM, 2); + + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); + + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records)); + + OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(ucd_record, other_case)); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(ucd_record, caseset)); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP3, 0); + CMPTO(SLJIT_EQUAL, TMP1, 0, char1_reg, 0, loop); + + add_jump(compiler, &no_match, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 2); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_caseless_sets)); + + caseless_loop = LABEL(); + OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(TMP2), 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, sizeof(uint32_t)); + OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP1, 0, char1_reg, 0); + JUMPTO(SLJIT_EQUAL, loop); + JUMPTO(SLJIT_LESS, caseless_loop); + + set_jumps(no_match, LABEL()); if (common->mode == PCRE2_JIT_COMPLETE) - add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1)); - else + JUMPHERE(partial); + + OP1(SLJIT_MOV, source_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr); + OP1(SLJIT_MOV, source_end_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw)); + OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw) * 2); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + + if (common->mode != PCRE2_JIT_COMPLETE) { - OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_LESS, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1); + JUMPHERE(partial); + OP1(SLJIT_MOV, source_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr); + OP1(SLJIT_MOV, source_end_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw)); + OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw) * 2); - add_jump(compiler, backtracks, JUMP(SLJIT_LESS)); - - nopartial = JUMP(SLJIT_NOT_EQUAL); - OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); check_partial(common, FALSE); add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(nopartial); } + + JUMPHERE(jump); + OP1(SLJIT_MOV, source_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr); + OP1(SLJIT_MOV, source_end_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw)); + OP1(SLJIT_MOV, char1_reg, 0, SLJIT_MEM1(SLJIT_SP), common->iref_ptr + sizeof(sljit_sw) * 2); + return; } else #endif /* SUPPORT_UNICODE */ @@ -8862,6 +10229,42 @@ if (common->optimized_cbracket[offset >> 1] == 0) return stacksize; } +static PCRE2_SPTR SLJIT_FUNC do_script_run(PCRE2_SPTR ptr, PCRE2_SPTR endptr) +{ + if (PRIV(script_run)(ptr, endptr, FALSE)) + return endptr; + return NULL; +} + +#ifdef SUPPORT_UNICODE + +static PCRE2_SPTR SLJIT_FUNC do_script_run_utf(PCRE2_SPTR ptr, PCRE2_SPTR endptr) +{ + if (PRIV(script_run)(ptr, endptr, TRUE)) + return endptr; + return NULL; +} + +#endif /* SUPPORT_UNICODE */ + +static SLJIT_INLINE void match_script_run_common(compiler_common *common, int private_data_ptr, backtrack_common *parent) +{ +DEFINE_COMPILER; + +SLJIT_ASSERT(TMP1 == SLJIT_R0 && STR_PTR == SLJIT_R1); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); +#ifdef SUPPORT_UNICODE +sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, + common->utf ? SLJIT_FUNC_OFFSET(do_script_run_utf) : SLJIT_FUNC_OFFSET(do_script_run)); +#else +sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_RET(SW) | SLJIT_ARG1(SW) | SLJIT_ARG2(SW), SLJIT_IMM, SLJIT_FUNC_OFFSET(do_script_run)); +#endif + +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); +add_jump(compiler, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); +} + /* Handling bracketed expressions is probably the most complex part. @@ -8997,7 +10400,7 @@ if (opcode == OP_CBRA || opcode == OP_SCBRA) BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; matchingpath += IMM2_SIZE; } -else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_ONCE || opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { /* Other brackets simply allocate the next entry. */ private_data_ptr = PRIVATE_DATA(ccbegin); @@ -9036,35 +10439,32 @@ if (bra == OP_BRAMINZERO) free_stack(common, 1); braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); } - else + else if (opcode == OP_ONCE || opcode >= OP_SBRA) { - if (opcode == OP_ONCE || opcode >= OP_SBRA) + jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + /* Nothing stored during the first run. */ + skip = JUMP(SLJIT_JUMP); + JUMPHERE(jump); + /* Checking zero-length iteration. */ + if (opcode != OP_ONCE || BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) { - jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - /* Nothing stored during the first run. */ - skip = JUMP(SLJIT_JUMP); - JUMPHERE(jump); - /* Checking zero-length iteration. */ - if (opcode != OP_ONCE || BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) - { - /* When we come from outside, private_data_ptr contains the previous STR_PTR. */ - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - } - else - { - /* Except when the whole stack frame must be saved. */ - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2)); - } - JUMPHERE(skip); + /* When we come from outside, private_data_ptr contains the previous STR_PTR. */ + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); } else { - jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - JUMPHERE(jump); + /* Except when the whole stack frame must be saved. */ + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2)); } + JUMPHERE(skip); + } + else + { + jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + JUMPHERE(jump); } } @@ -9081,7 +10481,7 @@ if (ket == OP_KETRMIN) if (ket == OP_KETRMAX) { rmax_label = LABEL(); - if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA && repeat_type == 0) + if (has_alternatives && opcode >= OP_BRA && opcode < OP_SBRA && repeat_type == 0) BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = rmax_label; } @@ -9185,7 +10585,7 @@ else if (opcode == OP_CBRA || opcode == OP_SCBRA) OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); } } -else if (opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { /* Saving the previous value. */ OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); @@ -9314,6 +10714,9 @@ if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) if (opcode == OP_ONCE) match_once_common(common, ket, BACKTRACK_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); +if (opcode == OP_SCRIPT_RUN) + match_script_run_common(common, private_data_ptr, backtrack); + stacksize = 0; if (repeat_type == OP_MINUPTO) { @@ -9383,13 +10786,15 @@ if (ket == OP_KETRMAX) if (opcode != OP_ONCE) free_stack(common, 1); } - else if (opcode == OP_ONCE || opcode >= OP_SBRA) + else if (opcode < OP_BRA || opcode >= OP_SBRA) { if (has_alternatives) BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); + /* Checking zero-length iteration. */ if (opcode != OP_ONCE) { + /* This case includes opcodes such as OP_SCRIPT_RUN. */ CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0, rmax_label); /* Drop STR_PTR for greedy plus quantifier. */ if (bra != OP_BRAZERO) @@ -9456,7 +10861,7 @@ if (opcode == OP_ONCE) /* We temporarily encode the needs_control_head in the lowest bit. Note: on the target architectures of SLJIT the ((x << 1) >> 1) returns the same value for small signed numbers (including negative numbers). */ - BACKTRACK_AS(bracket_backtrack)->u.framesize = (BACKTRACK_AS(bracket_backtrack)->u.framesize << 1) | (needs_control_head ? 1 : 0); + BACKTRACK_AS(bracket_backtrack)->u.framesize = (int)((unsigned)BACKTRACK_AS(bracket_backtrack)->u.framesize << 1) | (needs_control_head ? 1 : 0); } return cc + repeat_length; } @@ -9951,7 +11356,7 @@ if (exact > 1) #ifdef SUPPORT_UNICODE && !common->utf #endif - ) + && type != OP_ANYNL && type != OP_EXTUNI) { OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(exact)); add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER, TMP1, 0, STR_END, 0)); @@ -10634,6 +12039,7 @@ while (cc < ccend) break; case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRA: case OP_CBRA: case OP_COND: @@ -10787,14 +12193,14 @@ switch(opcode) if (CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit != 0) OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit); CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.chr, CURRENT_AS(char_iterator_backtrack)->matchingpath); - skip_char_back(common); + move_back(common, NULL, TRUE); CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP2, 0, label); } else { OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, base, offset1); - skip_char_back(common); + move_back(common, NULL, TRUE); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); } @@ -11240,6 +12646,9 @@ if (has_alternatives) compile_matchingpath(common, ccprev, cc, current); if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) return; + + if (opcode == OP_SCRIPT_RUN) + match_script_run_common(common, private_data_ptr, current); } /* Instructions after the current alternative is successfully matched. */ @@ -11368,7 +12777,7 @@ if (offset != 0) OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); } } -else if (opcode == OP_SBRA || opcode == OP_SCOND) +else if (opcode == OP_SCRIPT_RUN || opcode == OP_SBRA || opcode == OP_SCOND) { OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0)); free_stack(common, 1); @@ -11717,6 +13126,7 @@ while (current) break; case OP_ONCE: + case OP_SCRIPT_RUN: case OP_BRA: case OP_CBRA: case OP_COND: @@ -12016,6 +13426,9 @@ sljit_emit_fast_return(compiler, TMP2, 0); #undef COMPILE_BACKTRACKINGPATH #undef CURRENT_AS +#define PUBLIC_JIT_COMPILE_CONFIGURATION_OPTIONS \ + (PCRE2_JIT_INVALID_UTF) + static int jit_compile(pcre2_code *code, sljit_u32 mode) { pcre2_real_code *re = (pcre2_real_code *)code; @@ -12052,6 +13465,11 @@ common->re = re; common->name_table = (PCRE2_SPTR)((uint8_t *)re + sizeof(pcre2_real_code)); rootbacktrack.cc = common->name_table + re->name_count * re->name_entry_size; +#ifdef SUPPORT_UNICODE +common->invalid_utf = (mode & PCRE2_JIT_INVALID_UTF) != 0; +#endif /* SUPPORT_UNICODE */ +mode &= ~PUBLIC_JIT_COMPILE_CONFIGURATION_OPTIONS; + common->start = rootbacktrack.cc; common->read_only_data_head = NULL; common->fcc = tables + fcc_offset; @@ -12066,6 +13484,7 @@ switch(re->newline_convention) case PCRE2_NEWLINE_CRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; break; case PCRE2_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break; case PCRE2_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break; + case PCRE2_NEWLINE_NUL: common->newline = CHAR_NUL; break; default: return PCRE2_ERROR_INTERNAL; } common->nlmax = READ_CHAR_MAX; @@ -12117,6 +13536,8 @@ if (common->utf) common->bsr_nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL; common->bsr_nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL; } +else + common->invalid_utf = FALSE; #endif /* SUPPORT_UNICODE */ ccend = bracketend(common->start); @@ -12557,22 +13978,49 @@ if (common->utfreadchar != NULL) set_jumps(common->utfreadchar, LABEL()); do_utfreadchar(common); } -if (common->utfreadchar16 != NULL) - { - set_jumps(common->utfreadchar16, LABEL()); - do_utfreadchar16(common); - } if (common->utfreadtype8 != NULL) { set_jumps(common->utfreadtype8, LABEL()); do_utfreadtype8(common); } +if (common->utfpeakcharback != NULL) + { + set_jumps(common->utfpeakcharback, LABEL()); + do_utfpeakcharback(common); + } #endif /* PCRE2_CODE_UNIT_WIDTH == 8 */ +#if PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16 +if (common->utfreadchar_invalid != NULL) + { + set_jumps(common->utfreadchar_invalid, LABEL()); + do_utfreadchar_invalid(common); + } +if (common->utfreadnewline_invalid != NULL) + { + set_jumps(common->utfreadnewline_invalid, LABEL()); + do_utfreadnewline_invalid(common); + } +if (common->utfmoveback_invalid) + { + set_jumps(common->utfmoveback_invalid, LABEL()); + do_utfmoveback_invalid(common); + } +if (common->utfpeakcharback_invalid) + { + set_jumps(common->utfpeakcharback_invalid, LABEL()); + do_utfpeakcharback_invalid(common); + } +#endif /* PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16 */ if (common->getucd != NULL) { set_jumps(common->getucd, LABEL()); do_getucd(common); } +if (common->getucdtype != NULL) + { + set_jumps(common->getucdtype, LABEL()); + do_getucdtype(common); + } #endif /* SUPPORT_UNICODE */ SLJIT_FREE(common->optimized_cbracket, allocator_data); @@ -12644,7 +14092,7 @@ Returns: 0: success or (*NOJIT) was used */ #define PUBLIC_JIT_COMPILE_OPTIONS \ - (PCRE2_JIT_COMPLETE|PCRE2_JIT_PARTIAL_SOFT|PCRE2_JIT_PARTIAL_HARD) + (PCRE2_JIT_COMPLETE|PCRE2_JIT_PARTIAL_SOFT|PCRE2_JIT_PARTIAL_HARD|PCRE2_JIT_INVALID_UTF) PCRE2_EXP_DEFN int PCRE2_CALL_CONVENTION pcre2_jit_compile(pcre2_code *code, uint32_t options) @@ -12659,6 +14107,7 @@ return PCRE2_ERROR_JIT_BADOPTION; pcre2_real_code *re = (pcre2_real_code *)code; executable_functions *functions; +uint32_t excluded_options; int result; if (code == NULL) @@ -12673,21 +14122,24 @@ functions = (executable_functions *)re->executable_jit; if ((options & PCRE2_JIT_COMPLETE) != 0 && (functions == NULL || functions->executable_funcs[0] == NULL)) { - result = jit_compile(code, PCRE2_JIT_COMPLETE); + excluded_options = (PCRE2_JIT_PARTIAL_SOFT | PCRE2_JIT_PARTIAL_HARD); + result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } if ((options & PCRE2_JIT_PARTIAL_SOFT) != 0 && (functions == NULL || functions->executable_funcs[1] == NULL)) { - result = jit_compile(code, PCRE2_JIT_PARTIAL_SOFT); + excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_HARD); + result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } if ((options & PCRE2_JIT_PARTIAL_HARD) != 0 && (functions == NULL || functions->executable_funcs[2] == NULL)) { - result = jit_compile(code, PCRE2_JIT_PARTIAL_HARD); + excluded_options = (PCRE2_JIT_COMPLETE | PCRE2_JIT_PARTIAL_SOFT); + result = jit_compile(code, options & ~excluded_options); if (result != 0) return result; } diff --git a/thirdparty/pcre2/src/pcre2_jit_match.c b/thirdparty/pcre2/src/pcre2_jit_match.c index 5a66545bae3..eee038644dd 100644 --- a/thirdparty/pcre2/src/pcre2_jit_match.c +++ b/thirdparty/pcre2/src/pcre2_jit_match.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016 University of Cambridge + New API code Copyright (c) 2016-2018 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -152,8 +152,6 @@ else jit_stack = NULL; } -/* JIT only need two offsets for each ovector entry. Hence - the last 1/3 of the ovector will never be touched. */ max_oveccount = functions->top_bracket; if (oveccount > max_oveccount) @@ -173,7 +171,7 @@ else if (rc > (int)oveccount) rc = 0; match_data->code = re; -match_data->subject = subject; +match_data->subject = (rc >= 0 || rc == PCRE2_ERROR_PARTIAL)? subject : NULL; match_data->rc = rc; match_data->startchar = arguments.startchar_ptr - subject; match_data->leftchar = 0; diff --git a/thirdparty/pcre2/src/pcre2_maketables.c b/thirdparty/pcre2/src/pcre2_maketables.c index 537edba8c32..5921e907930 100644 --- a/thirdparty/pcre2/src/pcre2_maketables.c +++ b/thirdparty/pcre2/src/pcre2_maketables.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -114,17 +114,17 @@ test for alnum specially. */ memset(p, 0, cbit_length); for (i = 0; i < 256; i++) { - if (isdigit(i)) p[cbit_digit + i/8] |= 1 << (i&7); - if (isupper(i)) p[cbit_upper + i/8] |= 1 << (i&7); - if (islower(i)) p[cbit_lower + i/8] |= 1 << (i&7); - if (isalnum(i)) p[cbit_word + i/8] |= 1 << (i&7); - if (i == '_') p[cbit_word + i/8] |= 1 << (i&7); - if (isspace(i)) p[cbit_space + i/8] |= 1 << (i&7); - if (isxdigit(i))p[cbit_xdigit + i/8] |= 1 << (i&7); - if (isgraph(i)) p[cbit_graph + i/8] |= 1 << (i&7); - if (isprint(i)) p[cbit_print + i/8] |= 1 << (i&7); - if (ispunct(i)) p[cbit_punct + i/8] |= 1 << (i&7); - if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1 << (i&7); + if (isdigit(i)) p[cbit_digit + i/8] |= 1u << (i&7); + if (isupper(i)) p[cbit_upper + i/8] |= 1u << (i&7); + if (islower(i)) p[cbit_lower + i/8] |= 1u << (i&7); + if (isalnum(i)) p[cbit_word + i/8] |= 1u << (i&7); + if (i == '_') p[cbit_word + i/8] |= 1u << (i&7); + if (isspace(i)) p[cbit_space + i/8] |= 1u << (i&7); + if (isxdigit(i))p[cbit_xdigit + i/8] |= 1u << (i&7); + if (isgraph(i)) p[cbit_graph + i/8] |= 1u << (i&7); + if (isprint(i)) p[cbit_print + i/8] |= 1u << (i&7); + if (ispunct(i)) p[cbit_punct + i/8] |= 1u << (i&7); + if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1u << (i&7); } p += cbit_length; @@ -138,8 +138,8 @@ for (i = 0; i < 256; i++) int x = 0; if (isspace(i)) x += ctype_space; if (isalpha(i)) x += ctype_letter; + if (islower(i)) x += ctype_lcletter; if (isdigit(i)) x += ctype_digit; - if (isxdigit(i)) x += ctype_xdigit; if (isalnum(i) || i == '_') x += ctype_word; *p++ = x; } diff --git a/thirdparty/pcre2/src/pcre2_match.c b/thirdparty/pcre2/src/pcre2_match.c index 8741e1432d6..419561fd645 100644 --- a/thirdparty/pcre2/src/pcre2_match.c +++ b/thirdparty/pcre2/src/pcre2_match.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2015-2018 University of Cambridge + New API code Copyright (c) 2015-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -69,11 +69,12 @@ information, and fields within it. */ #define PUBLIC_MATCH_OPTIONS \ (PCRE2_ANCHORED|PCRE2_ENDANCHORED|PCRE2_NOTBOL|PCRE2_NOTEOL|PCRE2_NOTEMPTY| \ PCRE2_NOTEMPTY_ATSTART|PCRE2_NO_UTF_CHECK|PCRE2_PARTIAL_HARD| \ - PCRE2_PARTIAL_SOFT|PCRE2_NO_JIT) + PCRE2_PARTIAL_SOFT|PCRE2_NO_JIT|PCRE2_COPY_MATCHED_SUBJECT) #define PUBLIC_JIT_MATCH_OPTIONS \ (PCRE2_NO_UTF_CHECK|PCRE2_NOTBOL|PCRE2_NOTEOL|PCRE2_NOTEMPTY|\ - PCRE2_NOTEMPTY_ATSTART|PCRE2_PARTIAL_SOFT|PCRE2_PARTIAL_HARD) + PCRE2_NOTEMPTY_ATSTART|PCRE2_PARTIAL_SOFT|PCRE2_PARTIAL_HARD|\ + PCRE2_COPY_MATCHED_SUBJECT) /* Non-error returns from and within the match() function. Error returns are externally defined PCRE2_ERROR_xxx codes, which are all negative. */ @@ -1848,7 +1849,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (Fop == OP_CLASS) RRETURN(MATCH_NOMATCH); } else - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); } } else @@ -1870,7 +1871,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); } else #endif - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); } } @@ -1902,7 +1903,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (Fop == OP_CLASS) RRETURN(MATCH_NOMATCH); } else - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); } } else @@ -1927,7 +1928,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); } else #endif - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) RRETURN(MATCH_NOMATCH); } } /* Control never gets here */ @@ -1956,7 +1957,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); if (Fop == OP_CLASS) break; } else - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) break; + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) break; Feptr += len; } @@ -1993,7 +1994,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); } else #endif - if ((Lbyte_map[fc/8] & (1 << (fc&7))) == 0) break; + if ((Lbyte_map[fc/8] & (1u << (fc&7))) == 0) break; Feptr++; } @@ -4084,7 +4085,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); GETCHAR(fc, fptr); } lgb = UCD_GRAPHBREAK(fc); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + if ((PRIV(ucp_gbtable)[lgb] & (1u << rgb)) == 0) break; Feptr = fptr; rgb = lgb; } @@ -5014,6 +5015,7 @@ fprintf(stderr, "++ op=%d\n", *Fecode); must record a backtracking point and also set up a chained frame. */ case OP_ONCE: + case OP_SCRIPT_RUN: case OP_SBRA: Lframe_type = GF_NOCAPTURE | Fop; @@ -5526,6 +5528,14 @@ fprintf(stderr, "++ op=%d\n", *Fecode); case OP_ASSERTBACK_NOT: RRETURN(MATCH_MATCH); + /* At the end of a script run, apply the script-checking rules. This code + will never by exercised if Unicode support it not compiled, because in + that environment script runs cause an error at compile time. */ + + case OP_SCRIPT_RUN: + if (!PRIV(script_run)(P->eptr, Feptr, utf)) RRETURN(MATCH_NOMATCH); + break; + /* Whole-pattern recursion is coded as a recurse into group 0, so it won't be picked up here. Instead, we catch it when the OP_END is reached. Other recursion is handled here. */ @@ -6000,10 +6010,11 @@ pcre2_match(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, pcre2_match_context *mcontext) { int rc; +int was_zero_terminated = 0; const uint8_t *start_bits = NULL; - const pcre2_real_code *re = (const pcre2_real_code *)code; + BOOL anchored; BOOL firstline; BOOL has_first_cu = FALSE; @@ -6043,7 +6054,11 @@ mb->stack_frames = (heapframe *)stack_frames_vector; /* A length equal to PCRE2_ZERO_TERMINATED implies a zero-terminated subject string. */ -if (length == PCRE2_ZERO_TERMINATED) length = PRIV(strlen)(subject); +if (length == PCRE2_ZERO_TERMINATED) + { + length = PRIV(strlen)(subject); + was_zero_terminated = 1; + } end_subject = subject + length; /* Plausibility checks */ @@ -6158,6 +6173,17 @@ if (mcontext != NULL && mcontext->offset_limit != PCRE2_UNSET && (re->overall_options & PCRE2_USE_OFFSET_LIMIT) == 0) return PCRE2_ERROR_BADOFFSETLIMIT; +/* If the match data block was previously used with PCRE2_COPY_MATCHED_SUBJECT, +free the memory that was obtained. Set the field to NULL for no match cases. */ + +if ((match_data->flags & PCRE2_MD_COPIED_SUBJECT) != 0) + { + match_data->memctl.free((void *)match_data->subject, + match_data->memctl.memory_data); + match_data->flags &= ~PCRE2_MD_COPIED_SUBJECT; + } +match_data->subject = NULL; + /* If the pattern was successfully studied with JIT support, run the JIT executable instead of the rest of this function. Most options must be set at compile time for the JIT code to be usable. Fallback to the normal code path if @@ -6169,7 +6195,19 @@ if (re->executable_jit != NULL && (options & ~PUBLIC_JIT_MATCH_OPTIONS) == 0) { rc = pcre2_jit_match(code, subject, length, start_offset, options, match_data, mcontext); - if (rc != PCRE2_ERROR_JIT_BADOPTION) return rc; + if (rc != PCRE2_ERROR_JIT_BADOPTION) + { + if (rc >= 0 && (options & PCRE2_COPY_MATCHED_SUBJECT) != 0) + { + length = CU2BYTES(length + was_zero_terminated); + match_data->subject = match_data->memctl.malloc(length, + match_data->memctl.memory_data); + if (match_data->subject == NULL) return PCRE2_ERROR_NOMEMORY; + memcpy((void *)match_data->subject, subject, length); + match_data->flags |= PCRE2_MD_COPIED_SUBJECT; + } + return rc; + } } #endif @@ -6421,7 +6459,7 @@ for(;;) #if PCRE2_CODE_UNIT_WIDTH != 8 if (c > 255) c = 255; #endif - ok = (start_bits[c/8] & (1 << (c&7))) != 0; + ok = (start_bits[c/8] & (1u << (c&7))) != 0; } } if (!ok) @@ -6538,7 +6576,7 @@ for(;;) #if PCRE2_CODE_UNIT_WIDTH != 8 if (c > 255) c = 255; #endif - if ((start_bits[c/8] & (1 << (c&7))) != 0) break; + if ((start_bits[c/8] & (1u << (c&7))) != 0) break; start_match++; } @@ -6809,13 +6847,13 @@ if (mb->match_frames != mb->stack_frames) /* Fill in fields that are always returned in the match data. */ match_data->code = re; -match_data->subject = subject; match_data->mark = mb->mark; match_data->matchedby = PCRE2_MATCHEDBY_INTERPRETER; /* Handle a fully successful match. Set the return code to the number of captured strings, or 0 if there were too many to fit into the ovector, and then -set the remaining returned values before returning. */ +set the remaining returned values before returning. Make a copy of the subject +string if requested. */ if (rc == MATCH_MATCH) { @@ -6825,6 +6863,16 @@ if (rc == MATCH_MATCH) match_data->leftchar = mb->start_used_ptr - subject; match_data->rightchar = ((mb->last_used_ptr > mb->end_match_ptr)? mb->last_used_ptr : mb->end_match_ptr) - subject; + if ((options & PCRE2_COPY_MATCHED_SUBJECT) != 0) + { + length = CU2BYTES(length + was_zero_terminated); + match_data->subject = match_data->memctl.malloc(length, + match_data->memctl.memory_data); + if (match_data->subject == NULL) return PCRE2_ERROR_NOMEMORY; + memcpy((void *)match_data->subject, subject, length); + match_data->flags |= PCRE2_MD_COPIED_SUBJECT; + } + else match_data->subject = subject; return match_data->rc; } @@ -6838,10 +6886,14 @@ match_data->mark = mb->nomatch_mark; if (rc != MATCH_NOMATCH && rc != PCRE2_ERROR_PARTIAL) match_data->rc = rc; -/* Handle a partial match. */ +/* Handle a partial match. If a "soft" partial match was requested, searching +for a complete match will have continued, and the value of rc at this point +will be MATCH_NOMATCH. For a "hard" partial match, it will already be +PCRE2_ERROR_PARTIAL. */ else if (match_partial != NULL) { + match_data->subject = subject; match_data->ovector[0] = match_partial - subject; match_data->ovector[1] = end_subject - subject; match_data->startchar = match_partial - subject; diff --git a/thirdparty/pcre2/src/pcre2_match_data.c b/thirdparty/pcre2/src/pcre2_match_data.c index b297f326b55..ccc5f6740e0 100644 --- a/thirdparty/pcre2/src/pcre2_match_data.c +++ b/thirdparty/pcre2/src/pcre2_match_data.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2017 University of Cambridge + New API code Copyright (c) 2016-2018 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -63,6 +63,7 @@ yield = PRIV(memctl_malloc)( (pcre2_memctl *)gcontext); if (yield == NULL) return NULL; yield->oveccount = oveccount; +yield->flags = 0; return yield; } @@ -93,7 +94,12 @@ PCRE2_EXP_DEFN void PCRE2_CALL_CONVENTION pcre2_match_data_free(pcre2_match_data *match_data) { if (match_data != NULL) + { + if ((match_data->flags & PCRE2_MD_COPIED_SUBJECT) != 0) + match_data->memctl.free((void *)match_data->subject, + match_data->memctl.memory_data); match_data->memctl.free(match_data, match_data->memctl.memory_data); + } } diff --git a/thirdparty/pcre2/src/pcre2_script_run.c b/thirdparty/pcre2/src/pcre2_script_run.c new file mode 100644 index 00000000000..91a48330283 --- /dev/null +++ b/thirdparty/pcre2/src/pcre2_script_run.c @@ -0,0 +1,441 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Original API code Copyright (c) 1997-2012 University of Cambridge + New API code Copyright (c) 2016-2018 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* This module contains the function for checking a script run. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre2_internal.h" + + +/************************************************* +* Check script run * +*************************************************/ + +/* A script run is conceptually a sequence of characters all in the same +Unicode script. However, it isn't quite that simple. There are special rules +for scripts that are commonly used together, and also special rules for digits. +This function implements the appropriate checks, which is possible only when +PCRE2 is compiled with Unicode support. The function returns TRUE if there is +no Unicode support; however, it should never be called in that circumstance +because an error is given by pcre2_compile() if a script run is called for in a +version of PCRE2 compiled without Unicode support. + +Arguments: + pgr point to the first character + endptr point after the last character + utf TRUE if in UTF mode + +Returns: TRUE if this is a valid script run +*/ + +/* These dummy values must be less than the negation of the largest offset in +the PRIV(ucd_script_sets) vector, which is held in a 16-bit field in UCD +records (and is only likely to be a few hundred). */ + +#define SCRIPT_UNSET (-99999) +#define SCRIPT_HANPENDING (-99998) +#define SCRIPT_HANHIRAKATA (-99997) +#define SCRIPT_HANBOPOMOFO (-99996) +#define SCRIPT_HANHANGUL (-99995) +#define SCRIPT_LIST (-99994) + +#define INTERSECTION_LIST_SIZE 50 + +BOOL +PRIV(script_run)(PCRE2_SPTR ptr, PCRE2_SPTR endptr, BOOL utf) +{ +#ifdef SUPPORT_UNICODE +int require_script = SCRIPT_UNSET; +uint8_t intersection_list[INTERSECTION_LIST_SIZE]; +const uint8_t *require_list = NULL; +uint32_t require_digitset = 0; +uint32_t c; + +#if PCRE2_CODE_UNIT_WIDTH == 32 +(void)utf; /* Avoid compiler warning */ +#endif + +/* Any string containing fewer than 2 characters is a valid script run. */ + +if (ptr >= endptr) return TRUE; +GETCHARINCTEST(c, ptr); +if (ptr >= endptr) return TRUE; + +/* Scan strings of two or more characters, checking the Unicode characteristics +of each code point. We make use of the Script Extensions property. There is +special code for scripts that can be combined with characters from the Han +Chinese script. This may be used in conjunction with four other scripts in +these combinations: + +. Han with Hiragana and Katakana is allowed (for Japanese). +. Han with Bopomofo is allowed (for Taiwanese Mandarin). +. Han with Hangul is allowed (for Korean). + +If the first significant character's script is one of the four, the required +script type is immediately known. However, if the first significant +character's script is Han, we have to keep checking for a non-Han character. +Hence the SCRIPT_HANPENDING state. */ + +for (;;) + { + const ucd_record *ucd = GET_UCD(c); + int32_t scriptx = ucd->scriptx; + + /* If the script extension is Unknown, the string is not a valid script run. + Such characters can only form script runs of length one. */ + + if (scriptx == ucp_Unknown) return FALSE; + + /* A character whose script extension is Inherited is always accepted with + any script, and plays no further part in this testing. A character whose + script is Common is always accepted, but must still be tested for a digit + below. The scriptx value at this point is non-zero, because zero is + ucp_Unknown, tested for above. */ + + if (scriptx != ucp_Inherited) + { + if (scriptx != ucp_Common) + { + /* If the script extension value is positive, the character is not a mark + that can be used with many scripts. In the simple case we either set or + compare with the required script. However, handling the scripts that can + combine with Han are more complicated, as is the case when the previous + characters have been man-script marks. */ + + if (scriptx > 0) + { + switch(require_script) + { + /* Either the first significant character (require_script unset) or + after only Han characters. */ + + case SCRIPT_UNSET: + case SCRIPT_HANPENDING: + switch(scriptx) + { + case ucp_Han: + require_script = SCRIPT_HANPENDING; + break; + + case ucp_Hiragana: + case ucp_Katakana: + require_script = SCRIPT_HANHIRAKATA; + break; + + case ucp_Bopomofo: + require_script = SCRIPT_HANBOPOMOFO; + break; + + case ucp_Hangul: + require_script = SCRIPT_HANHANGUL; + break; + + /* Not a Han-related script. If expecting one, fail. Otherise set + the requirement to this script. */ + + default: + if (require_script == SCRIPT_HANPENDING) return FALSE; + require_script = scriptx; + break; + } + break; + + /* Previously encountered one of the "with Han" scripts. Check that + this character is appropriate. */ + + case SCRIPT_HANHIRAKATA: + if (scriptx != ucp_Han && scriptx != ucp_Hiragana && + scriptx != ucp_Katakana) + return FALSE; + break; + + case SCRIPT_HANBOPOMOFO: + if (scriptx != ucp_Han && scriptx != ucp_Bopomofo) return FALSE; + break; + + case SCRIPT_HANHANGUL: + if (scriptx != ucp_Han && scriptx != ucp_Hangul) return FALSE; + break; + + /* We have a list of scripts to check that is derived from one or + more previous characters. This is either one of the lists in + ucd_script_sets[] (for one previous character) or the intersection of + several lists for multiple characters. */ + + case SCRIPT_LIST: + { + const uint8_t *list; + for (list = require_list; *list != 0; list++) + { + if (*list == scriptx) break; + } + if (*list == 0) return FALSE; + } + + /* The rest of the string must be in this script, but we have to + allow for the Han complications. */ + + switch(scriptx) + { + case ucp_Han: + require_script = SCRIPT_HANPENDING; + break; + + case ucp_Hiragana: + case ucp_Katakana: + require_script = SCRIPT_HANHIRAKATA; + break; + + case ucp_Bopomofo: + require_script = SCRIPT_HANBOPOMOFO; + break; + + case ucp_Hangul: + require_script = SCRIPT_HANHANGUL; + break; + + default: + require_script = scriptx; + break; + } + break; + + /* This is the easy case when a single script is required. */ + + default: + if (scriptx != require_script) return FALSE; + break; + } + } /* End of handing positive scriptx */ + + /* If scriptx is negative, this character is a mark-type character that + has a list of permitted scripts. */ + + else + { + uint32_t chspecial; + const uint8_t *clist, *rlist; + const uint8_t *list = PRIV(ucd_script_sets) - scriptx; + + switch(require_script) + { + case SCRIPT_UNSET: + require_list = PRIV(ucd_script_sets) - scriptx; + require_script = SCRIPT_LIST; + break; + + /* An inspection of the Unicode 11.0.0 files shows that there are the + following types of Script Extension list that involve the Han, + Bopomofo, Hiragana, Katakana, and Hangul scripts: + + . Bopomofo + Han + . Han + Hiragana + Katakana + . Hiragana + Katakana + . Bopopmofo + Hangul + Han + Hiragana + Katakana + + The following code tries to make sense of this. */ + +#define FOUND_BOPOMOFO 1 +#define FOUND_HIRAGANA 2 +#define FOUND_KATAKANA 4 +#define FOUND_HANGUL 8 + + case SCRIPT_HANPENDING: + chspecial = 0; + for (; *list != 0; list++) + { + switch (*list) + { + case ucp_Bopomofo: chspecial |= FOUND_BOPOMOFO; break; + case ucp_Hiragana: chspecial |= FOUND_HIRAGANA; break; + case ucp_Katakana: chspecial |= FOUND_KATAKANA; break; + case ucp_Hangul: chspecial |= FOUND_HANGUL; break; + default: break; + } + } + + if (chspecial == 0) return FALSE; + + if (chspecial == FOUND_BOPOMOFO) + { + require_script = SCRIPT_HANBOPOMOFO; + } + else if (chspecial == (FOUND_HIRAGANA|FOUND_KATAKANA)) + { + require_script = SCRIPT_HANHIRAKATA; + } + + /* Otherwise it must be allowed with all of them, so remain in + the pending state. */ + + break; + + case SCRIPT_HANHIRAKATA: + for (; *list != 0; list++) + { + if (*list == ucp_Hiragana || *list == ucp_Katakana) break; + } + if (*list == 0) return FALSE; + break; + + case SCRIPT_HANBOPOMOFO: + for (; *list != 0; list++) + { + if (*list == ucp_Bopomofo) break; + } + if (*list == 0) return FALSE; + break; + + case SCRIPT_HANHANGUL: + for (; *list != 0; list++) + { + if (*list == ucp_Hangul) break; + } + if (*list == 0) return FALSE; + break; + + /* Previously encountered one or more characters that are allowed + with a list of scripts. Build the intersection of the required list + with this character's list in intersection_list[]. This code is + written so that it still works OK if the required list is already in + that vector. */ + + case SCRIPT_LIST: + { + int i = 0; + for (rlist = require_list; *rlist != 0; rlist++) + { + for (clist = list; *clist != 0; clist++) + { + if (*rlist == *clist) + { + intersection_list[i++] = *rlist; + break; + } + } + } + if (i == 0) return FALSE; /* No scripts in common */ + + /* If there's just one script in common, we can set it as the + unique required script. Otherwise, terminate the intersection list + and make it the required list. */ + + if (i == 1) + { + require_script = intersection_list[0]; + } + else + { + intersection_list[i] = 0; + require_list = intersection_list; + } + } + break; + + /* The previously set required script is a single script, not + Han-related. Check that it is in this character's list. */ + + default: + for (; *list != 0; list++) + { + if (*list == require_script) break; + } + if (*list == 0) return FALSE; + break; + } + } /* End of handling negative scriptx */ + } /* End of checking non-Common character */ + + /* The character is in an acceptable script. We must now ensure that all + decimal digits in the string come from the same set. Some scripts (e.g. + Common, Arabic) have more than one set of decimal digits. This code does + not allow mixing sets, even within the same script. The vector called + PRIV(ucd_digit_sets)[] contains, in its first element, the number of + following elements, and then, in ascending order, the code points of the + '9' characters in every set of 10 digits. Each set is identified by the + offset in the vector of its '9' character. An initial check of the first + value picks up ASCII digits quickly. Otherwise, a binary chop is used. */ + + if (ucd->chartype == ucp_Nd) + { + uint32_t digitset; + + if (c <= PRIV(ucd_digit_sets)[1]) digitset = 1; else + { + int mid; + int bot = 1; + int top = PRIV(ucd_digit_sets)[0]; + for (;;) + { + if (top <= bot + 1) /* <= rather than == is paranoia */ + { + digitset = top; + break; + } + mid = (top + bot) / 2; + if (c <= PRIV(ucd_digit_sets)[mid]) top = mid; else bot = mid; + } + } + + /* A required value of 0 means "unset". */ + + if (require_digitset == 0) require_digitset = digitset; + else if (digitset != require_digitset) return FALSE; + } /* End digit handling */ + } /* End checking non-Inherited character */ + + /* If we haven't yet got to the end, pick up the next character. */ + + if (ptr >= endptr) return TRUE; + GETCHARINCTEST(c, ptr); + } /* End checking loop */ + +#else /* NOT SUPPORT_UNICODE */ +(void)ptr; +(void)endptr; +(void)utf; +return TRUE; +#endif /* SUPPORT_UNICODE */ +} + +/* End of pcre2_script_run.c */ diff --git a/thirdparty/pcre2/src/pcre2_study.c b/thirdparty/pcre2/src/pcre2_study.c index acbf98b41b5..e883c2eb4c2 100644 --- a/thirdparty/pcre2/src/pcre2_study.c +++ b/thirdparty/pcre2/src/pcre2_study.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -54,7 +54,7 @@ collecting data (e.g. minimum matching length). */ /* Set a bit in the starting code unit bit map. */ -#define SET_BIT(c) re->start_bitmap[(c)/8] |= (1 << ((c)&7)) +#define SET_BIT(c) re->start_bitmap[(c)/8] |= (1u << ((c)&7)) /* Returns from set_start_bits() */ @@ -171,6 +171,7 @@ for (;;) /* Fall through */ case OP_ONCE: + case OP_SCRIPT_RUN: case OP_SBRA: case OP_BRAPOS: case OP_SBRAPOS: @@ -842,7 +843,7 @@ for (c = 0; c < table_limit; c++) if (table_limit == 32) return; for (c = 128; c < 256; c++) { - if ((re->tables[cbits_offset + c/8] & (1 << (c&7))) != 0) + if ((re->tables[cbits_offset + c/8] & (1u << (c&7))) != 0) { PCRE2_UCHAR buff[6]; (void)PRIV(ord2utf)(c, buff); @@ -1075,6 +1076,7 @@ do case OP_CBRAPOS: case OP_SCBRAPOS: case OP_ONCE: + case OP_SCRIPT_RUN: case OP_ASSERT: rc = set_start_bits(re, tcode, utf); if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; @@ -1505,11 +1507,11 @@ do for (c = 0; c < 16; c++) re->start_bitmap[c] |= classmap[c]; for (c = 128; c < 256; c++) { - if ((classmap[c/8] & (1 << (c&7))) != 0) + if ((classmap[c/8] & (1u << (c&7))) != 0) { - int d = (c >> 6) | 0xc0; /* Set bit for this starter */ - re->start_bitmap[d/8] |= (1 << (d&7)); /* and then skip on to the */ - c = (c & 0xc0) + 0x40 - 1; /* next relevant character. */ + int d = (c >> 6) | 0xc0; /* Set bit for this starter */ + re->start_bitmap[d/8] |= (1u << (d&7)); /* and then skip on to the */ + c = (c & 0xc0) + 0x40 - 1; /* next relevant character. */ } } } diff --git a/thirdparty/pcre2/src/pcre2_substitute.c b/thirdparty/pcre2/src/pcre2_substitute.c index ab8d10908aa..ec3dd66df97 100644 --- a/thirdparty/pcre2/src/pcre2_substitute.c +++ b/thirdparty/pcre2/src/pcre2_substitute.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -129,7 +129,7 @@ for (; ptr < ptrend; ptr++) ptr += 1; /* Must point after \ */ erc = PRIV(check_escape)(&ptr, ptrend, &ch, &errorcode, - code->overall_options, FALSE, NULL); + code->overall_options, code->extra_options, FALSE, NULL); ptr -= 1; /* Back to last code unit of escape */ if (errorcode != 0) { @@ -239,13 +239,17 @@ PCRE2_SIZE extra_needed = 0; PCRE2_SIZE buff_offset, buff_length, lengthleft, fraglength; PCRE2_SIZE *ovector; PCRE2_SIZE ovecsave[3]; +pcre2_substitute_callout_block scb; + +/* General initialization */ buff_offset = 0; lengthleft = buff_length = *blength; *blength = PCRE2_UNSET; ovecsave[0] = ovecsave[1] = ovecsave[2] = PCRE2_UNSET; -/* Partial matching is not valid. */ +/* Partial matching is not valid. This must come after setting *blength to +PCRE2_UNSET, so as not to imply an offset in the replacement. */ if ((options & (PCRE2_PARTIAL_HARD|PCRE2_PARTIAL_SOFT)) != 0) return PCRE2_ERROR_BADOPTION; @@ -264,6 +268,13 @@ if (match_data == NULL) ovector = pcre2_get_ovector_pointer(match_data); ovector_count = pcre2_get_ovector_count(match_data); +/* Fixed things in the callout block */ + +scb.version = 0; +scb.input = subject; +scb.output = (PCRE2_SPTR)buffer; +scb.ovector = ovector; + /* Find lengths of zero-terminated strings and the end of the replacement. */ if (length == PCRE2_ZERO_TERMINATED) length = PRIV(strlen)(subject); @@ -390,7 +401,7 @@ do rc = PCRE2_ERROR_INTERNAL_DUPMATCH; goto EXIT; } - + /* Count substitutions with a paranoid check for integer overflow; surely no real call to this function would ever hit this! */ @@ -401,11 +412,14 @@ do } subs++; - /* Copy the text leading up to the match. */ + /* Copy the text leading up to the match, and remember where the insert + begins and how many ovector pairs are set. */ if (rc == 0) rc = ovector_count; fraglength = ovector[0] - start_offset; CHECKMEMCPY(subject + start_offset, fraglength); + scb.output_offsets[0] = buff_offset; + scb.oveccount = rc; /* Process the replacement string. Literal mode is set by \Q, but only in extended mode when backslashes are being interpreted. In extended mode we @@ -421,7 +435,7 @@ do if (ptr >= repend) { - if (ptrstackptr <= 0) break; /* End of replacement string */ + if (ptrstackptr == 0) break; /* End of replacement string */ repend = ptrstack[--ptrstackptr]; ptr = ptrstack[--ptrstackptr]; continue; @@ -702,7 +716,7 @@ do { if (((code->tables + cbits_offset + ((forcecase > 0)? cbit_upper:cbit_lower) - )[ch/8] & (1 << (ch%8))) == 0) + )[ch/8] & (1u << (ch%8))) == 0) ch = (code->tables + fcc_offset)[ch]; } forcecase = forcecasereset; @@ -760,7 +774,7 @@ do ptr++; /* Point after \ */ rc = PRIV(check_escape)(&ptr, repend, &ch, &errorcode, - code->overall_options, FALSE, NULL); + code->overall_options, code->extra_options, FALSE, NULL); if (errorcode != 0) goto BADESCAPE; switch(rc) @@ -804,7 +818,7 @@ do { if (((code->tables + cbits_offset + ((forcecase > 0)? cbit_upper:cbit_lower) - )[ch/8] & (1 << (ch%8))) == 0) + )[ch/8] & (1u << (ch%8))) == 0) ch = (code->tables + fcc_offset)[ch]; } forcecase = forcecasereset; @@ -821,10 +835,37 @@ do } /* End handling a literal code unit */ } /* End of loop for scanning the replacement. */ - /* The replacement has been copied to the output. Save the details of this - match. See above for how this data is used. If we matched an empty string, do - the magic for global matches. Finally, update the start offset to point to - the rest of the subject string. */ + /* The replacement has been copied to the output, or its size has been + remembered. Do the callout if there is one and we have done an actual + replacement. */ + + if (!overflowed && mcontext != NULL && mcontext->substitute_callout != NULL) + { + scb.subscount = subs; + scb.output_offsets[1] = buff_offset; + rc = mcontext->substitute_callout(&scb, mcontext->substitute_callout_data); + + /* A non-zero return means cancel this substitution. Instead, copy the + matched string fragment. */ + + if (rc != 0) + { + PCRE2_SIZE newlength = scb.output_offsets[1] - scb.output_offsets[0]; + PCRE2_SIZE oldlength = ovector[1] - ovector[0]; + + buff_offset -= newlength; + lengthleft += newlength; + CHECKMEMCPY(subject + ovector[0], oldlength); + + /* A negative return means do not do any more. */ + + if (rc < 0) suboptions &= (~PCRE2_SUBSTITUTE_GLOBAL); + } + } + + /* Save the details of this match. See above for how this data is used. If we + matched an empty string, do the magic for global matches. Finally, update the + start offset to point to the rest of the subject string. */ ovecsave[0] = ovector[0]; ovecsave[1] = ovector[1]; diff --git a/thirdparty/pcre2/src/pcre2_tables.c b/thirdparty/pcre2/src/pcre2_tables.c index 83d6f9de55e..84019361fce 100644 --- a/thirdparty/pcre2/src/pcre2_tables.c +++ b/thirdparty/pcre2/src/pcre2_tables.c @@ -7,7 +7,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge - New API code Copyright (c) 2016-2018 University of Cambridge + New API code Copyright (c) 2016-2019 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -142,7 +142,7 @@ ucp_gbXX values defined in pcre2_ucp.h. These changed between Unicode versions code points. The left property selects a word from the table, and the right property selects a bit from that word like this: - PRIV(ucp_gbtable)[left-property] & (1 << right-property) + PRIV(ucp_gbtable)[left-property] & (1u << right-property) The value is non-zero if a grapheme break is NOT permitted between the relevant two code points. The breaking rules are as follows: @@ -183,25 +183,25 @@ are implementing). #define ESZ (1< + +#if TARGET_OS_OSX && defined(MAP_JIT) +#include +#endif /* TARGET_OS_OSX && MAP_JIT */ + +#ifdef MAP_JIT + +static SLJIT_INLINE int get_map_jit_flag() +{ +#if TARGET_OS_OSX + /* On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a version + of macOS where it's OK to have more than one JIT block. On non-macOS systems, returns + MAP_JIT if it is defined. */ + static int map_jit_flag = -1; + + /* The following code is thread safe because multiple initialization + sets map_jit_flag to the same value and the code has no side-effects. + Changing the kernel version witout system restart is (very) unlikely. */ + if (map_jit_flag == -1) { + struct utsname name; + + uname(&name); + + /* Kernel version for 10.14.0 (Mojave) */ + map_jit_flag = (atoi(name.release) >= 18) ? MAP_JIT : 0; + } + + return map_jit_flag; +#else /* !TARGET_OS_OSX */ + return MAP_JIT; +#endif /* TARGET_OS_OSX */ +} + +#endif /* MAP_JIT */ + +#endif /* __APPLE__ */ + static SLJIT_INLINE void* alloc_chunk(sljit_uw size) { void *retval; @@ -103,17 +143,17 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) int flags = MAP_PRIVATE | MAP_ANON; #ifdef MAP_JIT - flags |= MAP_JIT; + flags |= get_map_jit_flag(); #endif retval = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC, flags, -1, 0); -#else +#else /* !MAP_ANON */ if (dev_zero < 0) { if (open_dev_zero()) return NULL; } retval = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE, dev_zero, 0); -#endif +#endif /* MAP_ANON */ return (retval != MAP_FAILED) ? retval : NULL; } diff --git a/thirdparty/pcre2/src/sljit/sljitLir.c b/thirdparty/pcre2/src/sljit/sljitLir.c index 5bdddc10cff..ded9541b31b 100644 --- a/thirdparty/pcre2/src/sljit/sljitLir.c +++ b/thirdparty/pcre2/src/sljit/sljitLir.c @@ -201,15 +201,16 @@ # define IS_CALL 0x010 # define IS_BIT26_COND 0x020 # define IS_BIT16_COND 0x040 +# define IS_BIT23_COND 0x080 -# define IS_COND (IS_BIT26_COND | IS_BIT16_COND) +# define IS_COND (IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND) -# define PATCH_B 0x080 -# define PATCH_J 0x100 +# define PATCH_B 0x100 +# define PATCH_J 0x200 #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) -# define PATCH_ABS32 0x200 -# define PATCH_ABS48 0x400 +# define PATCH_ABS32 0x400 +# define PATCH_ABS48 0x800 #endif /* instruction types */ diff --git a/thirdparty/pcre2/src/sljit/sljitNativeARM_64.c b/thirdparty/pcre2/src/sljit/sljitNativeARM_64.c index 27af7414877..b015695c524 100644 --- a/thirdparty/pcre2/src/sljit/sljitNativeARM_64.c +++ b/thirdparty/pcre2/src/sljit/sljitNativeARM_64.c @@ -51,7 +51,7 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { 0, 0, 1, 2, 3, 4, 5, 6, 7 }; -#define W_OP (1 << 31) +#define W_OP (1u << 31) #define RD(rd) (reg_map[rd]) #define RT(rt) (reg_map[rt]) #define RN(rn) (reg_map[rn] << 5) @@ -560,7 +560,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s /* dst must be register, TMP_REG1 arg1 must be register, TMP_REG1, imm arg2 must be register, TMP_REG2, imm */ - sljit_ins inv_bits = (flags & INT_OP) ? (1 << 31) : 0; + sljit_ins inv_bits = (flags & INT_OP) ? W_OP : 0; sljit_ins inst_bits; sljit_s32 op = (flags & 0xffff); sljit_s32 reg; @@ -710,7 +710,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s return push_inst(compiler, ORR | RD(dst) | RN(TMP_ZERO) | RM(arg2)); case SLJIT_MOV_U8: SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); - return push_inst(compiler, (UBFM ^ (1 << 31)) | RD(dst) | RN(arg2) | (7 << 10)); + return push_inst(compiler, (UBFM ^ W_OP) | RD(dst) | RN(arg2) | (7 << 10)); case SLJIT_MOV_S8: SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); if (!(flags & INT_OP)) @@ -718,7 +718,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s return push_inst(compiler, (SBFM ^ inv_bits) | RD(dst) | RN(arg2) | (7 << 10)); case SLJIT_MOV_U16: SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); - return push_inst(compiler, (UBFM ^ (1 << 31)) | RD(dst) | RN(arg2) | (15 << 10)); + return push_inst(compiler, (UBFM ^ W_OP) | RD(dst) | RN(arg2) | (15 << 10)); case SLJIT_MOV_S16: SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); if (!(flags & INT_OP)) @@ -728,7 +728,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); if ((flags & INT_OP) && dst == arg2) return SLJIT_SUCCESS; - return push_inst(compiler, (ORR ^ (1 << 31)) | RD(dst) | RN(TMP_ZERO) | RM(arg2)); + return push_inst(compiler, (ORR ^ W_OP) | RD(dst) | RN(TMP_ZERO) | RM(arg2)); case SLJIT_MOV_S32: SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1); if ((flags & INT_OP) && dst == arg2) @@ -1080,7 +1080,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *comp SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op) { - sljit_ins inv_bits = (op & SLJIT_I32_OP) ? (1 << 31) : 0; + sljit_ins inv_bits = (op & SLJIT_I32_OP) ? W_OP : 0; CHECK_ERROR(); CHECK(check_sljit_emit_op0(compiler, op)); @@ -1360,7 +1360,7 @@ static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_comp sljit_ins inv_bits = (op & SLJIT_F32_OP) ? (1 << 22) : 0; if (GET_OPCODE(op) == SLJIT_CONV_S32_FROM_F64) - inv_bits |= (1 << 31); + inv_bits |= W_OP; if (src & SLJIT_MEM) { emit_fop_mem(compiler, (op & SLJIT_F32_OP) ? INT_SIZE : WORD_SIZE, TMP_FREG1, src, srcw); @@ -1382,7 +1382,7 @@ static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_comp sljit_ins inv_bits = (op & SLJIT_F32_OP) ? (1 << 22) : 0; if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32) - inv_bits |= (1 << 31); + inv_bits |= W_OP; if (src & SLJIT_MEM) { emit_op_mem(compiler, ((GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32) ? INT_SIZE : WORD_SIZE), TMP_REG1, src, srcw, TMP_REG1); @@ -1662,7 +1662,7 @@ static SLJIT_INLINE struct sljit_jump* emit_cmp_to0(struct sljit_compiler *compi sljit_s32 src, sljit_sw srcw) { struct sljit_jump *jump; - sljit_ins inv_bits = (type & SLJIT_I32_OP) ? (1 << 31) : 0; + sljit_ins inv_bits = (type & SLJIT_I32_OP) ? W_OP : 0; SLJIT_ASSERT((type & 0xff) == SLJIT_EQUAL || (type & 0xff) == SLJIT_NOT_EQUAL); ADJUST_LOCAL_OFFSET(src, srcw); @@ -1787,7 +1787,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compil sljit_s32 dst_reg, sljit_s32 src, sljit_sw srcw) { - sljit_ins inv_bits = (dst_reg & SLJIT_I32_OP) ? (1 << 31) : 0; + sljit_ins inv_bits = (dst_reg & SLJIT_I32_OP) ? W_OP : 0; sljit_ins cc; CHECK_ERROR(); diff --git a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_32.c b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_32.c index 094c9923bcd..ad970bf25a8 100644 --- a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_32.c +++ b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_32.c @@ -368,16 +368,21 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(!(flags & SRC2_IMM)); if (GET_FLAG_TYPE(op) != SLJIT_MUL_OVERFLOW) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) || (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) return push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst)); -#else +#else /* !SLJIT_MIPS_R1 && !SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, MULT | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#endif +#endif /* SLJIT_MIPS_R1 || SLJIT_MIPS_R6 */ } +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + FAIL_IF(push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst))); + FAIL_IF(push_inst(compiler, MUH | S(src1) | T(src2) | DA(EQUAL_FLAG), EQUAL_FLAG)); +#else /* !SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, MULT | S(src1) | T(src2), MOVABLE_INS)); FAIL_IF(push_inst(compiler, MFHI | DA(EQUAL_FLAG), EQUAL_FLAG)); FAIL_IF(push_inst(compiler, MFLO | D(dst), DR(dst))); +#endif /* SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, SRA | T(dst) | DA(OTHER_FLAG) | SH_IMM(31), OTHER_FLAG)); return push_inst(compiler, SUBU | SA(EQUAL_FLAG) | TA(OTHER_FLAG) | DA(OTHER_FLAG), OTHER_FLAG); diff --git a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_64.c b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_64.c index f841aef5ddc..a6a2bcc0c9a 100644 --- a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_64.c +++ b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_64.c @@ -459,19 +459,26 @@ static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sl SLJIT_ASSERT(!(flags & SRC2_IMM)); if (GET_FLAG_TYPE(op) != SLJIT_MUL_OVERFLOW) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + return push_inst(compiler, SELECT_OP(DMUL, MUL) | S(src1) | T(src2) | D(dst), DR(dst)); +#elif (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) if (op & SLJIT_I32_OP) return push_inst(compiler, MUL | S(src1) | T(src2) | D(dst), DR(dst)); FAIL_IF(push_inst(compiler, DMULT | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#else +#else /* !SLJIT_MIPS_R6 && !SLJIT_MIPS_R1 */ FAIL_IF(push_inst(compiler, SELECT_OP(DMULT, MULT) | S(src1) | T(src2), MOVABLE_INS)); return push_inst(compiler, MFLO | D(dst), DR(dst)); -#endif +#endif /* SLJIT_MIPS_R6 */ } +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + FAIL_IF(push_inst(compiler, SELECT_OP(DMUL, MUL) | S(src1) | T(src2) | D(dst), DR(dst))); + FAIL_IF(push_inst(compiler, SELECT_OP(DMUH, MUH) | S(src1) | T(src2) | DA(EQUAL_FLAG), EQUAL_FLAG)); +#else /* !SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, SELECT_OP(DMULT, MULT) | S(src1) | T(src2), MOVABLE_INS)); FAIL_IF(push_inst(compiler, MFHI | DA(EQUAL_FLAG), EQUAL_FLAG)); FAIL_IF(push_inst(compiler, MFLO | D(dst), DR(dst))); +#endif /* SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, SELECT_OP(DSRA32, SRA) | T(dst) | DA(OTHER_FLAG) | SH_IMM(31), OTHER_FLAG)); return push_inst(compiler, SELECT_OP(DSUBU, SUBU) | SA(EQUAL_FLAG) | TA(OTHER_FLAG) | DA(OTHER_FLAG), OTHER_FLAG); diff --git a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_common.c b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_common.c index 894e21304bb..e0d6a3f0854 100644 --- a/thirdparty/pcre2/src/sljit/sljitNativeMIPS_common.c +++ b/thirdparty/pcre2/src/sljit/sljitNativeMIPS_common.c @@ -27,17 +27,31 @@ /* Latest MIPS architecture. */ /* Automatically detect SLJIT_MIPS_R1 */ +#if (defined __mips_isa_rev) && (__mips_isa_rev >= 6) +#define SLJIT_MIPS_R6 1 +#endif + SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) { -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + +#if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) + return "MIPS32-R6" SLJIT_CPUINFO; +#else /* !SLJIT_CONFIG_MIPS_32 */ + return "MIPS64-R6" SLJIT_CPUINFO; +#endif /* SLJIT_CONFIG_MIPS_32 */ + +#elif (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) + #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) return "MIPS32-R1" SLJIT_CPUINFO; -#else +#else /* !SLJIT_CONFIG_MIPS_32 */ return "MIPS64-R1" SLJIT_CPUINFO; -#endif +#endif /* SLJIT_CONFIG_MIPS_32 */ + #else /* SLJIT_MIPS_R1 */ return "MIPS III" SLJIT_CPUINFO; -#endif +#endif /* SLJIT_MIPS_R6 */ } /* Length of an instruction word @@ -62,6 +76,7 @@ typedef sljit_u32 sljit_ins; #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1) #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2) +#define TMP_FREG3 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3) static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 5] = { 0, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 23, 22, 21, 20, 19, 18, 17, 16, 29, 4, 25, 31 @@ -69,14 +84,14 @@ static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 5] = { #if (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32) -static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { - 0, 0, 14, 2, 4, 6, 8, 12, 10 +static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { + 0, 0, 14, 2, 4, 6, 8, 12, 10, 16 }; #else -static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { - 0, 0, 13, 14, 15, 16, 17, 12, 18 +static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { + 0, 0, 13, 14, 15, 16, 17, 12, 18, 10 }; #endif @@ -102,6 +117,11 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define FR(dr) (freg_map[dr]) #define HI(opcode) ((opcode) << 26) #define LO(opcode) (opcode) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +/* CMP.cond.fmt */ +/* S = (20 << 21) D = (21 << 21) */ +#define CMP_FMT_S (20 << 21) +#endif /* SLJIT_MIPS_R6 */ /* S = (16 << 21) D = (17 << 21) */ #define FMT_S (16 << 21) #define FMT_D (17 << 21) @@ -114,8 +134,13 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define ANDI (HI(12)) #define B (HI(4)) #define BAL (HI(1) | (17 << 16)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define BC1EQZ (HI(17) | (9 << 21) | FT(TMP_FREG3)) +#define BC1NEZ (HI(17) | (13 << 21) | FT(TMP_FREG3)) +#else /* !SLJIT_MIPS_R6 */ #define BC1F (HI(17) | (8 << 21)) #define BC1T (HI(17) | (8 << 21) | (1 << 16)) +#endif /* SLJIT_MIPS_R6 */ #define BEQ (HI(4)) #define BGEZ (HI(1) | (1 << 16)) #define BGTZ (HI(7)) @@ -124,20 +149,42 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define BNE (HI(5)) #define BREAK (HI(0) | LO(13)) #define CFC1 (HI(17) | (2 << 21)) -#define C_UN_S (HI(17) | FMT_S | LO(49)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define C_UEQ_S (HI(17) | CMP_FMT_S | LO(3)) +#define C_ULE_S (HI(17) | CMP_FMT_S | LO(7)) +#define C_ULT_S (HI(17) | CMP_FMT_S | LO(5)) +#define C_UN_S (HI(17) | CMP_FMT_S | LO(1)) +#define C_FD (FD(TMP_FREG3)) +#else /* !SLJIT_MIPS_R6 */ #define C_UEQ_S (HI(17) | FMT_S | LO(51)) #define C_ULE_S (HI(17) | FMT_S | LO(55)) #define C_ULT_S (HI(17) | FMT_S | LO(53)) +#define C_UN_S (HI(17) | FMT_S | LO(49)) +#define C_FD (0) +#endif /* SLJIT_MIPS_R6 */ #define CVT_S_S (HI(17) | FMT_S | LO(32)) #define DADDIU (HI(25)) #define DADDU (HI(0) | LO(45)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define DDIV (HI(0) | (2 << 6) | LO(30)) +#define DDIVU (HI(0) | (2 << 6) | LO(31)) +#define DMOD (HI(0) | (3 << 6) | LO(30)) +#define DMODU (HI(0) | (3 << 6) | LO(31)) +#define DIV (HI(0) | (2 << 6) | LO(26)) +#define DIVU (HI(0) | (2 << 6) | LO(27)) +#define DMUH (HI(0) | (3 << 6) | LO(28)) +#define DMUHU (HI(0) | (3 << 6) | LO(29)) +#define DMUL (HI(0) | (2 << 6) | LO(28)) +#define DMULU (HI(0) | (2 << 6) | LO(29)) +#else /* !SLJIT_MIPS_R6 */ #define DDIV (HI(0) | LO(30)) #define DDIVU (HI(0) | LO(31)) #define DIV (HI(0) | LO(26)) #define DIVU (HI(0) | LO(27)) -#define DIV_S (HI(17) | FMT_S | LO(3)) #define DMULT (HI(0) | LO(28)) #define DMULTU (HI(0) | LO(29)) +#endif /* SLJIT_MIPS_R6 */ +#define DIV_S (HI(17) | FMT_S | LO(3)) #define DSLL (HI(0) | LO(56)) #define DSLL32 (HI(0) | LO(60)) #define DSLLV (HI(0) | LO(20)) @@ -151,18 +198,34 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define J (HI(2)) #define JAL (HI(3)) #define JALR (HI(0) | LO(9)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define JR (HI(0) | LO(9)) +#else /* !SLJIT_MIPS_R6 */ #define JR (HI(0) | LO(8)) +#endif /* SLJIT_MIPS_R6 */ #define LD (HI(55)) #define LUI (HI(15)) #define LW (HI(35)) #define MFC1 (HI(17)) +#if !(defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) #define MFHI (HI(0) | LO(16)) #define MFLO (HI(0) | LO(18)) +#else /* SLJIT_MIPS_R6 */ +#define MOD (HI(0) | (3 << 6) | LO(26)) +#define MODU (HI(0) | (3 << 6) | LO(27)) +#endif /* !SLJIT_MIPS_R6 */ #define MOV_S (HI(17) | FMT_S | LO(6)) #define MTC1 (HI(17) | (4 << 21)) -#define MUL_S (HI(17) | FMT_S | LO(2)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define MUH (HI(0) | (3 << 6) | LO(24)) +#define MUHU (HI(0) | (3 << 6) | LO(25)) +#define MUL (HI(0) | (2 << 6) | LO(24)) +#define MULU (HI(0) | (2 << 6) | LO(25)) +#else /* !SLJIT_MIPS_R6 */ #define MULT (HI(0) | LO(24)) #define MULTU (HI(0) | LO(25)) +#endif /* SLJIT_MIPS_R6 */ +#define MUL_S (HI(17) | FMT_S | LO(2)) #define NEG_S (HI(17) | FMT_S | LO(7)) #define NOP (HI(0) | LO(0)) #define NOR (HI(0) | LO(39)) @@ -188,14 +251,18 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = { #define XOR (HI(0) | LO(38)) #define XORI (HI(14)) -#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) +#if (defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) || (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) #define CLZ (HI(28) | LO(32)) +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#define DCLZ (LO(18)) +#else /* !SLJIT_MIPS_R6 */ #define DCLZ (HI(28) | LO(36)) #define MOVF (HI(0) | (0 << 16) | LO(1)) #define MOVN (HI(0) | LO(11)) #define MOVT (HI(0) | (1 << 16) | LO(1)) #define MOVZ (HI(0) | LO(10)) #define MUL (HI(28) | LO(2)) +#endif /* SLJIT_MIPS_R6 */ #define PREF (HI(51)) #define PREFX (HI(19) | LO(15)) #define SEB (HI(31) | (16 << 6) | LO(32)) @@ -234,7 +301,13 @@ static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins, sljit static SLJIT_INLINE sljit_ins invert_branch(sljit_s32 flags) { - return (flags & IS_BIT26_COND) ? (1 << 26) : (1 << 16); + if (flags & IS_BIT26_COND) + return (1 << 26); +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + if (flags & IS_BIT23_COND) + return (1 << 23); +#endif /* SLJIT_MIPS_R6 */ + return (1 << 16); } static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset) @@ -1075,34 +1148,62 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compile return push_inst(compiler, NOP, UNMOVABLE_INS); case SLJIT_LMUL_UW: case SLJIT_LMUL_SW: +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) + FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMULU : DMUL) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); + FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMUHU : DMUH) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); +#else /* !SLJIT_CONFIG_MIPS_64 */ + FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? MULU : MUL) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); + FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? MUHU : MUH) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); +#endif /* SLJIT_CONFIG_MIPS_64 */ + FAIL_IF(push_inst(compiler, ADDU_W | S(TMP_REG3) | TA(0) | D(SLJIT_R0), DR(SLJIT_R0))); + return push_inst(compiler, ADDU_W | S(TMP_REG1) | TA(0) | D(SLJIT_R1), DR(SLJIT_R1)); +#else /* !SLJIT_MIPS_R6 */ #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? DMULTU : DMULT) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); -#else +#else /* !SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, (op == SLJIT_LMUL_UW ? MULTU : MULT) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); -#endif +#endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, MFLO | D(SLJIT_R0), DR(SLJIT_R0))); return push_inst(compiler, MFHI | D(SLJIT_R1), DR(SLJIT_R1)); +#endif /* SLJIT_MIPS_R6 */ case SLJIT_DIVMOD_UW: case SLJIT_DIVMOD_SW: case SLJIT_DIV_UW: case SLJIT_DIV_SW: SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments); +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) +#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) + if (int_op) { + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? MODU : MOD) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); + } + else { + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DDIVU : DDIV) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DMODU : DMOD) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); + } +#else /* !SLJIT_CONFIG_MIPS_64 */ + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG3), DR(TMP_REG3))); + FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? MODU : MOD) | S(SLJIT_R0) | T(SLJIT_R1) | D(TMP_REG1), DR(TMP_REG1))); +#endif /* SLJIT_CONFIG_MIPS_64 */ + FAIL_IF(push_inst(compiler, ADDU_W | S(TMP_REG3) | TA(0) | D(SLJIT_R0), DR(SLJIT_R0))); + return (op >= SLJIT_DIV_UW) ? SLJIT_SUCCESS : push_inst(compiler, ADDU_W | S(TMP_REG1) | TA(0) | D(SLJIT_R1), DR(SLJIT_R1)); +#else /* !SLJIT_MIPS_R6 */ #if !(defined SLJIT_MIPS_R1 && SLJIT_MIPS_R1) FAIL_IF(push_inst(compiler, NOP, UNMOVABLE_INS)); FAIL_IF(push_inst(compiler, NOP, UNMOVABLE_INS)); -#endif - +#endif /* !SLJIT_MIPS_R1 */ #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64) if (int_op) FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); else FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DDIVU : DDIV) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); -#else +#else /* !SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, ((op | 0x2) == SLJIT_DIV_UW ? DIVU : DIV) | S(SLJIT_R0) | T(SLJIT_R1), MOVABLE_INS)); -#endif - +#endif /* SLJIT_CONFIG_MIPS_64 */ FAIL_IF(push_inst(compiler, MFLO | D(SLJIT_R0), DR(SLJIT_R0))); return (op >= SLJIT_DIV_UW) ? SLJIT_SUCCESS : push_inst(compiler, MFHI | D(SLJIT_R1), DR(SLJIT_R1)); +#endif /* SLJIT_MIPS_R6 */ } return SLJIT_SUCCESS; @@ -1408,8 +1509,7 @@ static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compile inst = C_UN_S; break; } - - return push_inst(compiler, inst | FMT(op) | FT(src2) | FS(src1), UNMOVABLE_INS); + return push_inst(compiler, inst | FMT(op) | FT(src2) | FS(src1) | C_FD, UNMOVABLE_INS); } SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op, @@ -1608,16 +1708,30 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compi flags = IS_BIT26_COND; \ delay_check = src; +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + +#define BR_T() \ + inst = BC1NEZ; \ + flags = IS_BIT23_COND; \ + delay_check = FCSR_FCC; +#define BR_F() \ + inst = BC1EQZ; \ + flags = IS_BIT23_COND; \ + delay_check = FCSR_FCC; + +#else /* !SLJIT_MIPS_R6 */ + #define BR_T() \ inst = BC1T | JUMP_LENGTH; \ flags = IS_BIT16_COND; \ delay_check = FCSR_FCC; - #define BR_F() \ inst = BC1F | JUMP_LENGTH; \ flags = IS_BIT16_COND; \ delay_check = FCSR_FCC; +#endif /* SLJIT_MIPS_R6 */ + SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type) { struct sljit_jump *jump; @@ -1927,7 +2041,11 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *co case SLJIT_GREATER_EQUAL_F64: case SLJIT_UNORDERED_F64: case SLJIT_ORDERED_F64: +#if (defined SLJIT_MIPS_R6 && SLJIT_MIPS_R6) + FAIL_IF(push_inst(compiler, MFC1 | TA(dst_ar) | FS(TMP_FREG3), dst_ar)); +#else /* !SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, CFC1 | TA(dst_ar) | DA(FCSR_REG), dst_ar)); +#endif /* SLJIT_MIPS_R6 */ FAIL_IF(push_inst(compiler, SRL | TA(dst_ar) | DA(dst_ar) | SH_IMM(23), dst_ar)); FAIL_IF(push_inst(compiler, ANDI | SA(dst_ar) | TA(dst_ar) | IMM(1), dst_ar)); src_ar = dst_ar; diff --git a/thirdparty/pcre2/src/sljit/sljitNativePPC_common.c b/thirdparty/pcre2/src/sljit/sljitNativePPC_common.c index 5ef4ac96c4a..b34e3965ed2 100644 --- a/thirdparty/pcre2/src/sljit/sljitNativePPC_common.c +++ b/thirdparty/pcre2/src/sljit/sljitNativePPC_common.c @@ -42,7 +42,7 @@ typedef sljit_u32 sljit_ins; #include #endif -#if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN) +#if (defined _CALL_ELF && _CALL_ELF == 2) #define SLJIT_PASS_ENTRY_ADDR_TO_CALL 1 #endif From 2e98ecd95f434ce790a3ed35a5b614fcfc1d09a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 11 Jul 2019 10:34:40 +0200 Subject: [PATCH 008/113] tinyexr: Sync with upstream a685e33 (cherry picked from commit b5b3aa920b55f5d3969818ba04e82257f3014f81) --- COPYRIGHT.txt | 2 +- thirdparty/tinyexr/tinyexr.h | 206 +++++++++++++++++++++++++++-------- 2 files changed, 161 insertions(+), 47 deletions(-) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index a41a869e2b7..b26a851f984 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -371,7 +371,7 @@ License: Expat Files: ./thirdparty/tinyexr/ Comment: TinyEXR -Copyright: 2014-2018, Syoyo Fujita +Copyright: 2014-2019, Syoyo Fujita 2002, Industrial Light & Magic, a division of Lucas Digital Ltd. LLC License: BSD-3-clause diff --git a/thirdparty/tinyexr/tinyexr.h b/thirdparty/tinyexr/tinyexr.h index 3c193918505..f22163738fe 100644 --- a/thirdparty/tinyexr/tinyexr.h +++ b/thirdparty/tinyexr/tinyexr.h @@ -1,5 +1,5 @@ /* -Copyright (c) 2014 - 2018, Syoyo Fujita and many contributors. +Copyright (c) 2014 - 2019, Syoyo Fujita and many contributors. All rights reserved. Redistribution and use in source and binary forms, with or without @@ -276,7 +276,8 @@ extern int LoadEXR(float **out_rgba, int *width, int *height, // @deprecated { to be removed. } // Simple wrapper API for ParseEXRHeaderFromFile. // checking given file is a EXR file(by just look up header) -// @return TINYEXR_SUCCEES for EXR image, TINYEXR_ERROR_INVALID_HEADER for others +// @return TINYEXR_SUCCEES for EXR image, TINYEXR_ERROR_INVALID_HEADER for +// others extern int IsEXR(const char *filename); // @deprecated { to be removed. } @@ -469,9 +470,10 @@ extern int LoadEXRFromMemory(float **out_rgba, int *width, int *height, #include #include #include -#include #include +//#include // debug + #include #include #include @@ -2537,10 +2539,10 @@ tinfl_status tinfl_decompress(tinfl_decompressor *r, tinfl_status status = TINFL_STATUS_FAILED; mz_uint32 num_bits, dist, counter, num_extra; tinfl_bit_buf_t bit_buf; - const mz_uint8 *pIn_buf_cur = pIn_buf_next, - *const pIn_buf_end = pIn_buf_next + *pIn_buf_size; - mz_uint8 *pOut_buf_cur = pOut_buf_next, - *const pOut_buf_end = pOut_buf_next + *pOut_buf_size; + const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = + pIn_buf_next + *pIn_buf_size; + mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = + pOut_buf_next + *pOut_buf_size; size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 @@ -2957,8 +2959,9 @@ void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, tinfl_status status = tinfl_decompress( &decomp, (const mz_uint8 *)pSrc_buf + src_buf_ofs, &src_buf_size, (mz_uint8 *)pBuf, pBuf ? (mz_uint8 *)pBuf + *pOut_len : NULL, - &dst_buf_size, (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | - TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); + &dst_buf_size, + (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | + TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); if ((status < 0) || (status == TINFL_STATUS_NEEDS_MORE_INPUT)) { MZ_FREE(pBuf); *pOut_len = 0; @@ -3011,9 +3014,8 @@ int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8 *)pIn_buf + in_buf_ofs, &in_buf_size, pDict, pDict + dict_ofs, &dst_buf_size, - (flags & - ~(TINFL_FLAG_HAS_MORE_INPUT | - TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))); + (flags & ~(TINFL_FLAG_HAS_MORE_INPUT | + TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))); in_buf_ofs += in_buf_size; if ((dst_buf_size) && (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size, pPut_buf_user))) @@ -3138,7 +3140,9 @@ static const mz_uint8 s_tdefl_large_dist_extra[128] = { // Radix sorts tdefl_sym_freq[] array by 16-bit key m_key. Returns ptr to sorted // values. -typedef struct { mz_uint16 m_key, m_sym_index; } tdefl_sym_freq; +typedef struct { + mz_uint16 m_key, m_sym_index; +} tdefl_sym_freq; static tdefl_sym_freq *tdefl_radix_sort_syms(mz_uint num_syms, tdefl_sym_freq *pSyms0, tdefl_sym_freq *pSyms1) { @@ -5282,9 +5286,10 @@ mz_bool mz_zip_reader_file_stat(mz_zip_archive *pZip, mz_uint file_index, n = MZ_READ_LE16(p + MZ_ZIP_CDH_COMMENT_LEN_OFS); n = MZ_MIN(n, MZ_ZIP_MAX_ARCHIVE_FILE_COMMENT_SIZE - 1); pStat->m_comment_size = n; - memcpy(pStat->m_comment, p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + - MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + - MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS), + memcpy(pStat->m_comment, + p + MZ_ZIP_CENTRAL_DIR_HEADER_SIZE + + MZ_READ_LE16(p + MZ_ZIP_CDH_FILENAME_LEN_OFS) + + MZ_READ_LE16(p + MZ_ZIP_CDH_EXTRA_LEN_OFS), n); pStat->m_comment[n] = '\0'; @@ -7008,6 +7013,11 @@ static void swap2(unsigned short *val) { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wunused-function" #endif + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif static void cpy4(int *dst_val, const int *src_val) { unsigned char *dst = reinterpret_cast(dst_val); const unsigned char *src = reinterpret_cast(src_val); @@ -7037,11 +7047,14 @@ static void cpy4(float *dst_val, const float *src_val) { dst[2] = src[2]; dst[3] = src[3]; } - #ifdef __clang__ #pragma clang diagnostic pop #endif +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + static void swap4(unsigned int *val) { #ifdef MINIZ_LITTLE_ENDIAN (void)val; @@ -7696,7 +7709,8 @@ static int rleUncompress(int inLength, int maxLength, const signed char in[], int count = -(static_cast(*in++)); inLength -= count + 1; - if (0 > (maxLength -= count)) return 0; + // Fixes #116: Add bounds check to in buffer. + if ((0 > (maxLength -= count)) || (inLength < 0)) return 0; memcpy(out, in, count); out += count; @@ -7790,13 +7804,19 @@ static void CompressRle(unsigned char *dst, } } -static void DecompressRle(unsigned char *dst, +static bool DecompressRle(unsigned char *dst, const unsigned long uncompressed_size, const unsigned char *src, unsigned long src_size) { if (uncompressed_size == src_size) { // Data is not compressed(Issue 40). memcpy(dst, src, src_size); - return; + return true; + } + + // Workaround for issue #112. + // TODO(syoyo): Add more robust out-of-bounds check in `rleUncompress`. + if (src_size <= 2) { + return false; } std::vector tmpBuf(uncompressed_size); @@ -7805,8 +7825,9 @@ static void DecompressRle(unsigned char *dst, static_cast(uncompressed_size), reinterpret_cast(src), reinterpret_cast(&tmpBuf.at(0))); - assert(ret == static_cast(uncompressed_size)); - (void)ret; + if (ret != static_cast(uncompressed_size)) { + return false; + } // // Apply EXR-specific? postprocess. Grabbed from OpenEXR's @@ -7845,6 +7866,8 @@ static void DecompressRle(unsigned char *dst, break; } } + + return true; } #if TINYEXR_USE_PIZ @@ -8556,7 +8579,7 @@ static bool hufUnpackEncTable( int lc = 0; for (; im <= iM; im++) { - if (p - *pcode > ni) { + if (p - *pcode >= ni) { return false; } @@ -8854,7 +8877,7 @@ static bool getCode(int po, int rlc, long long &c, int &lc, const char *&in, if (out + cs > oe) return false; // Bounds check for safety - // Issue 100. + // Issue 100. if ((out - 1) < ob) return false; unsigned short s = out[-1]; @@ -9348,6 +9371,10 @@ static bool DecompressPiz(unsigned char *outPtr, const unsigned char *inPtr, tinyexr::cpy4(&length, reinterpret_cast(ptr)); ptr += sizeof(int); + if (size_t((ptr - inPtr) + length) > inLen) { + return false; + } + std::vector tmpBuffer(tmpBufSize); hufUncompress(reinterpret_cast(ptr), length, &tmpBuffer); @@ -9642,8 +9669,9 @@ static bool DecodePixelData(/* out */ unsigned char **out_images, reinterpret_cast(&outBuf.at(0)), data_ptr, tmpBufLen, data_len, static_cast(num_channels), channels, width, num_lines); - assert(ret); - (void)ret; + if (!ret) { + return false; + } // For PIZ_COMPRESSION: // pixel sample data for channel 0 for scanline 0 @@ -9688,16 +9716,18 @@ static bool DecodePixelData(/* out */ unsigned char **out_images, } else { // HALF -> FLOAT FP32 f32 = half_to_float(hf); float *image = reinterpret_cast(out_images)[c]; + size_t offset = 0; if (line_order == 0) { - image += (static_cast(line_no) + v) * + offset = (static_cast(line_no) + v) * static_cast(x_stride) + u; } else { - image += static_cast( + offset = static_cast( (height - 1 - (line_no + static_cast(v)))) * static_cast(x_stride) + u; } + image += offset; *image = f32.f; } } @@ -9824,16 +9854,19 @@ static bool DecodePixelData(/* out */ unsigned char **out_images, } else { // HALF -> FLOAT tinyexr::FP32 f32 = half_to_float(hf); float *image = reinterpret_cast(out_images)[c]; + size_t offset = 0; if (line_order == 0) { - image += (static_cast(line_no) + v) * + offset = (static_cast(line_no) + v) * static_cast(x_stride) + u; } else { - image += (static_cast(height) - 1U - + offset = (static_cast(height) - 1U - (static_cast(line_no) + v)) * static_cast(x_stride) + u; } + image += offset; + *image = f32.f; } } @@ -9906,10 +9939,15 @@ static bool DecodePixelData(/* out */ unsigned char **out_images, pixel_data_size); unsigned long dstLen = static_cast(outBuf.size()); - assert(dstLen > 0); - tinyexr::DecompressRle(reinterpret_cast(&outBuf.at(0)), + if (dstLen == 0) { + return false; + } + + if (!tinyexr::DecompressRle(reinterpret_cast(&outBuf.at(0)), dstLen, data_ptr, - static_cast(data_len)); + static_cast(data_len))) { + return false; + } // For RLE_COMPRESSION: // pixel sample data for channel 0 for scanline 0 @@ -10739,12 +10777,28 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, if ((data_width < 0) || (data_height < 0)) { if (err) { std::stringstream ss; - ss << "Invalid data width or data height: " << data_width << ", " << data_height << std::endl; + ss << "Invalid data width or data height: " << data_width << ", " + << data_height << std::endl; (*err) += ss.str(); } return TINYEXR_ERROR_INVALID_DATA; } + // Do not allow too large data_width and data_height. header invalid? + { + const int threshold = 1024 * 8192; // heuristics + if ((data_width > threshold) || (data_height > threshold)) { + if (err) { + std::stringstream ss; + ss << "data_with or data_height too large. data_width: " << data_width + << ", " + << "data_height = " << data_height << std::endl; + (*err) += ss.str(); + } + return TINYEXR_ERROR_INVALID_DATA; + } + } + size_t num_blocks = offsets.size(); std::vector channel_offset_list; @@ -10762,6 +10816,25 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, bool invalid_data = false; // TODO(LTE): Use atomic lock for MT safety. if (exr_header->tiled) { + // value check + if (exr_header->tile_size_x < 0) { + if (err) { + std::stringstream ss; + ss << "Invalid tile size x : " << exr_header->tile_size_x << "\n"; + (*err) += ss.str(); + } + return TINYEXR_ERROR_INVALID_HEADER; + } + + if (exr_header->tile_size_y < 0) { + if (err) { + std::stringstream ss; + ss << "Invalid tile size y : " << exr_header->tile_size_y << "\n"; + (*err) += ss.str(); + } + return TINYEXR_ERROR_INVALID_HEADER; + } + size_t num_tiles = offsets.size(); // = # of blocks exr_image->tiles = static_cast( @@ -10840,12 +10913,17 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, } } else { // scanline format - // Don't allow too large image(256GB * pixel_data_size or more). Workaround for #104. - size_t data_len = size_t(data_width) * size_t(data_height) * size_t(num_channels); - if ((data_len == 0) || (data_len >= 0x4000000000)) { + // Don't allow too large image(256GB * pixel_data_size or more). Workaround + // for #104. + size_t total_data_len = + size_t(data_width) * size_t(data_height) * size_t(num_channels); + const bool total_data_len_overflown = sizeof(void*) == 8 ? (total_data_len >= 0x4000000000) : false; + if ((total_data_len == 0) || total_data_len_overflown ) { if (err) { std::stringstream ss; - ss << "Image data size is zero or too large: width = " << data_width << ", height = " << data_height << ", channels = " << num_channels << std::endl; + ss << "Image data size is zero or too large: width = " << data_width + << ", height = " << data_height << ", channels = " << num_channels + << std::endl; (*err) += ss.str(); } return TINYEXR_ERROR_INVALID_DATA; @@ -10880,12 +10958,21 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, if (size_t(data_len) > data_size) { invalid_data = true; + + } else if ((line_no > (2 << 20)) || (line_no < -(2 << 20))) { + // Too large value. Assume this is invalid + // 2**20 = 1048576 = heuristic value. + invalid_data = true; + } else if (data_len == 0) { + // TODO(syoyo): May be ok to raise the threshold for example `data_len + // < 4` + invalid_data = true; } else { + // line_no may be negative. int end_line_no = (std::min)(line_no + num_scanline_blocks, (exr_header->data_window[3] + 1)); int num_lines = end_line_no - line_no; - // assert(num_lines > 0); if (num_lines <= 0) { invalid_data = true; @@ -10894,7 +10981,16 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, data_ptr += 8; // Adjust line_no with data_window.bmin.y - line_no -= exr_header->data_window[1]; + + // overflow check + tinyexr_int64 lno = static_cast(line_no) - static_cast(exr_header->data_window[1]); + if (lno > std::numeric_limits::max()) { + line_no = -1; // invalid + } else if (lno < -std::numeric_limits::max()) { + line_no = -1; // invalid + } else { + line_no -= exr_header->data_window[1]; + } if (line_no < 0) { invalid_data = true; @@ -10919,6 +11015,10 @@ static int DecodeChunk(EXRImage *exr_image, const EXRHeader *exr_header, } if (invalid_data) { + if (err) { + std::stringstream ss; + (*err) += "Invalid data found when decoding pixels.\n"; + } return TINYEXR_ERROR_INVALID_DATA; } @@ -10995,7 +11095,7 @@ static int DecodeEXRImage(EXRImage *exr_image, const EXRHeader *exr_header, int data_width = exr_header->data_window[2] - exr_header->data_window[0]; if (data_width >= std::numeric_limits::max()) { // Issue 63 - tinyexr::SetErrorMessage("Invalid data window value", err); + tinyexr::SetErrorMessage("Invalid data width value", err); return TINYEXR_ERROR_INVALID_DATA; } data_width++; @@ -11008,10 +11108,23 @@ static int DecodeEXRImage(EXRImage *exr_image, const EXRHeader *exr_header, data_height++; if ((data_width < 0) || (data_height < 0)) { - tinyexr::SetErrorMessage("data window or data height is negative.", err); + tinyexr::SetErrorMessage("data width or data height is negative.", err); return TINYEXR_ERROR_INVALID_DATA; } + // Do not allow too large data_width and data_height. header invalid? + { + const int threshold = 1024 * 8192; // heuristics + if (data_width > threshold) { + tinyexr::SetErrorMessage("data width too large.", err); + return TINYEXR_ERROR_INVALID_DATA; + } + if (data_height > threshold) { + tinyexr::SetErrorMessage("data height too large.", err); + return TINYEXR_ERROR_INVALID_DATA; + } + } + // Read offset tables. size_t num_blocks = 0; @@ -11190,7 +11303,6 @@ int LoadEXR(float **out_rgba, int *width, int *height, const char *filename, static_cast(exr_image.height))); if (exr_header.tiled) { - for (int it = 0; it < exr_image.num_tiles; it++) { for (int j = 0; j < exr_header.tile_size_y; j++) { for (int i = 0; i < exr_header.tile_size_x; i++) { @@ -11325,7 +11437,7 @@ int IsEXR(const char *filename) { if (ret != TINYEXR_SUCCESS) { return TINYEXR_ERROR_INVALID_HEADER; } - + return TINYEXR_SUCCESS; } @@ -11434,7 +11546,6 @@ int LoadEXRFromMemory(float **out_rgba, int *width, int *height, static_cast(exr_image.height))); if (exr_header.tiled) { - for (int it = 0; it < exr_image.num_tiles; it++) { for (int j = 0; j < exr_header.tile_size_y; j++) { for (int i = 0; i < exr_header.tile_size_x; i++) { @@ -12115,7 +12226,7 @@ size_t SaveEXRImageToMemory(const EXRImage *exr_image, sizeof(tinyexr::tinyexr_uint64) * static_cast(num_blocks)); } - if ( memory.size() == 0 ) { + if (memory.size() == 0) { tinyexr::SetErrorMessage("Output memory size is zero", err); return 0; } @@ -12286,6 +12397,9 @@ int LoadDeepEXR(DeepImage *deep_image, const char *filename, const char **err) { size_t marker_size; if (!tinyexr::ReadAttribute(&attr_name, &attr_type, &data, &marker_size, marker, size)) { + std::stringstream ss; + ss << "Failed to parse attribute\n"; + tinyexr::SetErrorMessage(ss.str(), err); return TINYEXR_ERROR_INVALID_DATA; } marker += marker_size; From b9b2e80addd0d3c01891c6782ab50366309e1dc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Sat, 20 Jul 2019 20:47:07 +0200 Subject: [PATCH 009/113] zstd: Update to upstream version 1.4.1 (cherry picked from commit b81fa47f3882ef7d91baa9fa1351ea504fb019ca) --- thirdparty/README.md | 2 +- thirdparty/zstd/common/compiler.h | 7 + thirdparty/zstd/common/zstd_internal.h | 64 +++- thirdparty/zstd/compress/zstd_compress.c | 292 +++++++++++------- .../zstd/compress/zstd_compress_internal.h | 104 +++++-- thirdparty/zstd/compress/zstd_double_fast.c | 92 +++--- thirdparty/zstd/compress/zstd_fast.c | 49 ++- thirdparty/zstd/compress/zstd_lazy.c | 15 +- thirdparty/zstd/compress/zstd_ldm.c | 2 +- thirdparty/zstd/compress/zstd_opt.c | 77 +++-- thirdparty/zstd/compress/zstdmt_compress.c | 25 +- thirdparty/zstd/compress/zstdmt_compress.h | 1 + thirdparty/zstd/decompress/zstd_decompress.c | 21 +- .../zstd/decompress/zstd_decompress_block.c | 29 +- thirdparty/zstd/zstd.h | 76 +++-- 15 files changed, 567 insertions(+), 289 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index 66af7720223..54c207b2a63 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -537,7 +537,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.0 +- Version: 1.4.1 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 0836e3ed27a..87bf51ae8c8 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -127,6 +127,13 @@ } \ } +/* vectorization */ +#if !defined(__clang__) && defined(__GNUC__) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +#else +# define DONT_VECTORIZE +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 31f756ab581..81b16eac2ea 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -34,7 +34,6 @@ #endif #include "xxhash.h" /* XXH_reset, update, digest */ - #if defined (__cplusplus) extern "C" { #endif @@ -193,19 +192,72 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG; * Shared functions to include for inlining *********************************************/ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } + #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } +#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } + +#define WILDCOPY_OVERLENGTH 8 +#define VECLEN 16 + +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst, + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; /*! ZSTD_wildcopy() : * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); + + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } +} + +/*! ZSTD_wildcopy_16min() : + * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +{ + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + + assert(length >= 8); + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } } MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 2e163c8bf3d..1476512580b 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -385,6 +385,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_lcm_uncompressed; return bounds; + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -452,6 +457,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: default: return 0; } @@ -497,6 +503,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: break; default: RETURN_ERROR(parameter_unsupported); @@ -671,6 +678,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -692,13 +705,13 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->compressionLevel; break; case ZSTD_c_windowLog : - *value = CCtxParams->cParams.windowLog; + *value = (int)CCtxParams->cParams.windowLog; break; case ZSTD_c_hashLog : - *value = CCtxParams->cParams.hashLog; + *value = (int)CCtxParams->cParams.hashLog; break; case ZSTD_c_chainLog : - *value = CCtxParams->cParams.chainLog; + *value = (int)CCtxParams->cParams.chainLog; break; case ZSTD_c_searchLog : *value = CCtxParams->cParams.searchLog; @@ -773,6 +786,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_ldmHashRateLog : *value = CCtxParams->ldmParams.hashRateLog; break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -930,12 +946,12 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); - BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); - BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); - BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); - BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); - BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength); + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); return 0; } @@ -951,7 +967,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams) if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ } -# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) CLAMP(ZSTD_c_windowLog, cParams.windowLog); CLAMP(ZSTD_c_chainLog, cParams.chainLog); CLAMP(ZSTD_c_hashLog, cParams.hashLog); @@ -1282,15 +1298,14 @@ static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) } /*! ZSTD_invalidateMatchState() - * Invalidate all the matches in the match finder tables. - * Requires nextSrc and base to be set (can be NULL). + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). */ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) { ZSTD_window_clear(&ms->window); ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; ms->loadedDictEnd = 0; ms->opt.litLengthSum = 0; /* force reset of btopt stats */ ms->dictMatchState = NULL; @@ -1327,15 +1342,17 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pl typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; + static void* ZSTD_reset_matchState(ZSTD_matchState_t* ms, void* ptr, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, U32 const forCCtx) + ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -1349,7 +1366,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_invalidateMatchState(ms); /* opt parser space */ - if (forCCtx && (cParams->strategy >= ZSTD_btopt)) { + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); ms->opt.litFreq = (unsigned*)ptr; ms->opt.litLengthFreq = ms->opt.litFreq + (1< (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + #define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large * during at least this number of times, @@ -1388,7 +1418,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_CCtx_params params, - U64 pledgedSrcSize, + U64 const pledgedSrcSize, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { @@ -1400,13 +1430,21 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (ZSTD_equivalentParams(zc->appliedParams, params, zc->inBuffSize, zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); + zbuff, pledgedSrcSize) ) { + DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) + if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { + DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", + zc->appliedParams.cParams.windowLog, zc->blockSize); + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + /* prefer a reset, faster than a rescale */ + ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); + } return ZSTD_continueCCtx(zc, params, pledgedSrcSize); - } } + } } } DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); if (params.ldmParams.enableLdm) { @@ -1449,7 +1487,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB", + DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", zc->workSpaceSize >> 10, neededSpace >> 10); @@ -1491,7 +1529,10 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32; + ptr = ZSTD_reset_matchState(&zc->blockState.matchState, + zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, + ¶ms.cParams, + crp, ZSTD_resetTarget_CCtx); /* ldm hash table */ /* initialize bucketOffsets table later for pointer alignment */ @@ -1509,8 +1550,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, ptr, ¶ms.cParams, crp, /* forCCtx */ 1); - /* sequences storage */ zc->seqStore.maxNbSeq = maxNbSeq; zc->seqStore.sequencesStart = (seqDef*)ptr; @@ -1587,15 +1626,14 @@ static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, * handled in _enforceMaxDist */ } -static size_t ZSTD_resetCCtx_byAttachingCDict( - ZSTD_CCtx* cctx, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - U64 pledgedSrcSize, - ZSTD_buffered_policy_e zbuff) +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) { - { - const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + { const ZSTD_compressionParameters* const cdict_cParams = &cdict->matchState.cParams; unsigned const windowLog = params.cParams.windowLog; assert(windowLog != 0); /* Resize working context table params for input only, since the dict @@ -1607,8 +1645,7 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } - { - const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { @@ -1625,9 +1662,9 @@ static size_t ZSTD_resetCCtx_byAttachingCDict( cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } + /* loadedDictEnd is expressed within the referential of the active context */ cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; - } - } + } } cctx->dictID = cdict->dictID; @@ -1681,7 +1718,6 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } @@ -1761,7 +1797,6 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; dstMatchState->window = srcMatchState->window; dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; - dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3; dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; } dstCCtx->dictID = srcCCtx->dictID; @@ -1831,16 +1866,15 @@ static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const /*! ZSTD_reduceIndex() : * rescale all indexes to avoid future overflow (indexes are U32) */ -static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) { - ZSTD_matchState_t* const ms = &zc->blockState.matchState; - { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; + { U32 const hSize = (U32)1 << params->cParams.hashLog; ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); } - if (zc->appliedParams.cParams.strategy != ZSTD_fast) { - U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog; - if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2) + if (params->cParams.strategy != ZSTD_fast) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); else ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); @@ -2524,6 +2558,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); @@ -2532,6 +2567,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* seqHead : flags for FSE encoding type */ seqHead = op++; + assert(op <= oend); /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); @@ -2555,6 +2591,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (LLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for Offsets */ { unsigned max = MaxOff; @@ -2577,6 +2614,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (Offtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } /* build CTable for MatchLengths */ { unsigned max = MaxML; @@ -2597,6 +2635,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (MLtype == set_compressed) lastNCount = op; op += countSize; + assert(op <= oend); } } *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); @@ -2610,6 +2649,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, longOffsets, bmi2); FORWARD_IF_ERROR(bitstreamSize); op += bitstreamSize; + assert(op <= oend); /* zstd versions <= 1.3.4 mistakenly report corruption when * FSE_readNCount() receives a buffer < 4 bytes. * Fixed by https://github.com/facebook/zstd/pull/1146. @@ -2721,30 +2761,24 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthID = 0; } -static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) { ZSTD_matchState_t* const ms = &zc->blockState.matchState; - size_t cSize; - DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)ms->window.dictLimit, (unsigned)ms->nextToUpdate); + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); - cSize = 0; - goto out; /* don't even attempt compression below a certain srcSize */ + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ } ZSTD_resetSeqStore(&(zc->seqStore)); /* required for optimal parser to read stats from dictionary */ ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* tell the optimal parser how we expect to compress literals */ ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; - /* a gap between an attached dict and the current window is not safe, * they must remain adjacent, * and when that stops being the case, the dict must be unset */ @@ -2798,6 +2832,21 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); } } + return ZSTDbss_compress; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t cSize; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, @@ -2826,6 +2875,25 @@ out: } +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +{ + if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { + U32 const maxDist = (U32)1 << params->cParams.windowLog; + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(ms, params, correction); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + + /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2844,7 +2912,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, BYTE* const ostart = (BYTE*)dst; BYTE* op = ostart; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; - assert(cctx->appliedParams.cParams.windowLog <= 31); + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) @@ -2859,19 +2927,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } - ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; { size_t cSize = ZSTD_compressBlock_internal(cctx, @@ -2899,7 +2958,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, } } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; - return op-ostart; + return (size_t)(op-ostart); } @@ -2991,6 +3050,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); + assert(fhSize <= dstCapacity); dstCapacity -= fhSize; dst = (char*)dst + fhSize; cctx->stage = ZSTDcs_ongoing; @@ -3007,18 +3067,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) { - U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); - U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src); - ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); - if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; - else ms->nextToUpdate -= correction; - ms->loadedDictEnd = 0; - ms->dictMatchState = NULL; - } + ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); @@ -3074,7 +3123,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) { - const BYTE* const ip = (const BYTE*) src; + const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; ZSTD_window_update(&ms->window, src, srcSize); @@ -3085,32 +3134,42 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, if (srcSize <= HASH_READ_SIZE) return 0; - switch(params->cParams.strategy) - { - case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, dtlm); - break; - case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, dtlm); - break; + while (iend - ip > HASH_READ_SIZE) { + size_t const remaining = iend - ip; + size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); + const BYTE* const ichunk = ip + chunk; - case ZSTD_greedy: - case ZSTD_lazy: - case ZSTD_lazy2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); - break; + ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); - case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ - case ZSTD_btopt: - case ZSTD_btultra: - case ZSTD_btultra2: - if (srcSize >= HASH_READ_SIZE) - ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); - break; + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, ichunk, dtlm); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, ichunk, dtlm); + break; - default: - assert(0); /* not possible : not a valid strategy id */ + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (chunk >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE); + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + if (chunk >= HASH_READ_SIZE) + ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ip = ichunk; } ms->nextToUpdate = (U32)(iend - ms->window.base); @@ -3297,12 +3356,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, ZSTDcrp_continue, zbuff) ); - { - size_t const dictID = ZSTD_compress_insertDictionary( + { size_t const dictID = ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); - assert(dictID <= (size_t)(U32)-1); + assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; } return 0; @@ -3555,10 +3613,10 @@ static size_t ZSTD_initCDict_internal( /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState( - &cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, ZSTDcrp_continue, /* forCCtx */ 0); + { void* const end = ZSTD_reset_matchState(&cdict->matchState, + (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, + &cParams, + ZSTDcrp_continue, ZSTD_resetTarget_CDict); assert(end == (char*)cdict->workspace + cdict->workspaceSize); (void)end; } @@ -4068,7 +4126,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, case zcss_flush: DEBUGLOG(5, "flush stage"); { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; - size_t const flushed = ZSTD_limitCopy(op, oend-op, + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), zcs->outBuff + zcs->outBuffFlushedSize, toFlush); DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); @@ -4262,7 +4320,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; - size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); return toFlush; diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index cc3cbb9da9f..5495899be35 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -33,13 +33,13 @@ extern "C" { ***************************************/ #define kSearchStrength 8 #define HASH_READ_SIZE 8 -#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted". +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". It could be confused for a real successor at index "1", if sorted as larger than its predecessor. It's not a big deal though : candidate will just be sorted again. Additionally, candidate position 1 will be lost. But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. - The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy - Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ /*-************************************* @@ -128,21 +128,20 @@ typedef struct { BYTE const* base; /* All regular indexes relative to this position */ BYTE const* dictBase; /* extDict indexes relative to this position */ U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more data */ + U32 lowLimit; /* below that point, no more valid data */ } ZSTD_window_t; typedef struct ZSTD_matchState_t ZSTD_matchState_t; struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */ U32 nextToUpdate; /* index from which to continue table update */ - U32 nextToUpdate3; /* index from which to continue table update */ U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32* hashTable; U32* hashTable3; U32* chainTable; optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t * dictMatchState; + const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; }; @@ -195,6 +194,9 @@ struct ZSTD_CCtx_params_s { int compressionLevel; int forceWindow; /* force back-references to respect limit of * 1<maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ @@ -564,6 +566,9 @@ MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 /*-************************************* * Round buffer management ***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif /* Max current allowed */ #define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) /* Maximum chunk size before overflow correction needs to be called again */ @@ -675,31 +680,49 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * Updates lowLimit so that: * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd * - * This allows a simple check that index >= lowLimit to see if index is valid. - * This must be called before a block compression call, with srcEnd as the block - * source end. + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. * - * If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit. - * This is because dictionaries are allowed to be referenced as long as the last - * byte of the dictionary is in the window, but once they are out of range, - * they cannot be referenced. If loadedDictEndPtr is NULL, we use - * loadedDictEnd == 0. + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. * - * In normal dict mode, the dict is between lowLimit and dictLimit. In - * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary - * is below them. forceWindow and dictMatchState are therefore incompatible. + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. */ MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window, - void const* srcEnd, - U32 maxDist, - U32* loadedDictEndPtr, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base); - U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u", - (unsigned)blockEndIdx, (unsigned)maxDist); + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ if (blockEndIdx > maxDist + loadedDictEnd) { U32 const newLowLimit = blockEndIdx - maxDist; if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; @@ -708,10 +731,31 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, (unsigned)window->dictLimit, (unsigned)window->lowLimit); window->dictLimit = window->lowLimit; } - if (loadedDictEndPtr) - *loadedDictEndPtr = 0; - if (dictMatchStatePtr) - *dictMatchStatePtr = NULL; + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. */ +MEM_STATIC void +ZSTD_checkDictValidity(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) { + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; } } diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 47faf6d641d..5957255d904 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -43,8 +43,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } - } + } } } @@ -63,7 +62,10 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 prefixLowestIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowestValid = ms->window.dictLimit; + const U32 maxDistance = 1U << cParams->windowLog; + const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -95,8 +97,15 @@ size_t ZSTD_compressBlock_doubleFast_generic( dictCParams->chainLog : hBitsS; const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart); + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic"); + assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState); + /* if a dictionary is attached, it must be within window range */ + if (dictMode == ZSTD_dictMatchState) { + assert(lowestValid + maxDistance >= endIndex); + } + /* init */ ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { @@ -138,7 +147,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -147,7 +156,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); goto _match_stored; } @@ -170,8 +179,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( offset = (U32)(current - dictMatchIndexL - dictIndexDelta); while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ goto _match_found; - } - } + } } if (matchIndexS > prefixLowestIndex) { /* check prefix short match */ @@ -186,16 +194,14 @@ size_t ZSTD_compressBlock_doubleFast_generic( if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { goto _search_next_long; - } - } + } } ip += ((ip-anchor) >> kSearchStrength) + 1; continue; _search_next_long: - { - size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; const BYTE* matchL3 = base + matchIndexL3; @@ -221,9 +227,7 @@ _search_next_long: offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta); while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ goto _match_found; - } - } - } + } } } /* if no long +1 match, explore the short match we found */ if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) { @@ -242,7 +246,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -250,11 +254,14 @@ _match_stored: anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } /* check immediate repcode */ if (dictMode == ZSTD_dictMatchState) { @@ -278,8 +285,7 @@ _match_stored: continue; } break; - } - } + } } if (dictMode == ZSTD_noDict) { while ( (ip <= ilimit) @@ -294,14 +300,15 @@ _match_stored: ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ - } } } } + } } } + } /* while (ip < ilimit) */ /* save reps for next block */ rep[0] = offset_1 ? offset_1 : offsetSaved; rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -360,10 +367,15 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* anchor = istart; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; - const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; const BYTE* const prefixStart = base + prefixStartIndex; - const U32 dictStartIndex = ms->window.lowLimit; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; @@ -371,6 +383,10 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); @@ -396,7 +412,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -407,7 +423,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -432,23 +448,27 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; continue; } } - /* found a match : store it */ + /* move to next sequence start */ ip += mLength; anchor = ip; if (ip <= ilimit) { - /* Fill Table */ - hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; - hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; - hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = current+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + /* check immediate repcode */ while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); @@ -475,7 +495,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index ed997b441c1..a05b8a47f14 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -13,7 +13,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm) + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; @@ -41,6 +42,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -58,7 +60,10 @@ size_t ZSTD_compressBlock_fast_generic( const BYTE* ip0 = istart; const BYTE* ip1; const BYTE* anchor = istart; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validStartIndex = ms->window.dictLimit; + const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; @@ -165,7 +170,7 @@ _match: /* Requires: ip0, match0, offcode */ rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -222,8 +227,15 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); const U32 dictHLog = dictCParams->hashLog; - /* otherwise, we would get index underflow when translating a dict index - * into a local index */ + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + /* ensure there will be no no underflow + * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ @@ -251,7 +263,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -271,7 +283,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -286,7 +298,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -327,7 +339,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( rep[1] = offset_2 ? offset_2 : offsetSaved; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } size_t ZSTD_compressBlock_fast_dictMatchState( @@ -366,15 +378,24 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; const BYTE* anchor = istart; - const U32 dictStartIndex = ms->window.lowLimit; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 validLow = ms->window.lowLimit; + const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow; + const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; - const U32 prefixStartIndex = ms->window.dictLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const dictEnd = dictBase + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); + /* Search Loop */ while (ip < ilimit) { /* < instead of <=, because (ip+1) */ const size_t h = ZSTD_hashPtr(ip, hlog, mls); @@ -394,7 +415,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -410,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } /* found a match : store it */ @@ -445,7 +466,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 53f998a4374..94d906c01f1 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -83,7 +83,10 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, U32* largerPtr = smallerPtr + 1; U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ U32 dummy32; /* to be nullified at the end */ - U32 const windowLow = ms->window.lowLimit; + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", current, dictLimit, windowLow); @@ -239,7 +242,9 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const windowLow = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -490,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic ( const U32 dictLimit = ms->window.dictLimit; const BYTE* const prefixStart = base + dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; - const U32 lowLimit = ms->window.lowLimit; const U32 current = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowValid = ms->window.lowLimit; + const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid; const U32 minChain = current > chainSize ? current - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; @@ -653,7 +660,6 @@ size_t ZSTD_compressBlock_lazy_generic( /* init */ ip += (dictAndPrefixLength == 0); - ms->nextToUpdate3 = ms->nextToUpdate; if (dictMode == ZSTD_noDict) { U32 const maxRep = (U32)(ip - prefixLowest); if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; @@ -933,7 +939,6 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( U32 offset_1 = rep[0], offset_2 = rep[1]; /* init */ - ms->nextToUpdate3 = ms->nextToUpdate; ip += (ip == prefixStart); /* Match Loop */ diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 784d20f3ab7..3dcf86e6e8a 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -447,7 +447,7 @@ size_t ZSTD_ldm_generateSequences( if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) { U32 const ldmHSize = 1U << params->hashLog; U32 const correction = ZSTD_window_correctOverflow( - &ldmState->window, /* cycleLog */ 0, maxDist, src); + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); } /* 2. We enforce the maximum offset allowed. diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index efb69d3267d..e32e542e021 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -255,13 +255,13 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP * to provide a cost which is directly comparable to a match ending at same position */ static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel) { - if (optPtr->priceType >= zop_predef) return WEIGHT(litLength, optLevel); + if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel); /* dynamic statistics */ { U32 const llCode = ZSTD_LLcode(litLength); - int const contribution = (LL_bits[llCode] * BITCOST_MULTIPLIER) - + WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ - - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER) + + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */ + - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel); #if 1 return contribution; #else @@ -278,7 +278,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe const optState_t* const optPtr, int optLevel) { - int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel) + ZSTD_litLengthContribution(litLength, optPtr, optLevel); return contribution; } @@ -372,13 +372,15 @@ MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) /* Update hashTable3 up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ -static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip) +static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) { U32* const hashTable3 = ms->hashTable3; U32 const hashLog3 = ms->hashLog3; const BYTE* const base = ms->window.base; - U32 idx = ms->nextToUpdate3; - U32 const target = ms->nextToUpdate3 = (U32)(ip - base); + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); assert(hashLog3 > 0); @@ -387,6 +389,7 @@ static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* idx++; } + *nextToUpdate3 = target; return hashTable3[hash3]; } @@ -503,9 +506,11 @@ static U32 ZSTD_insertBt1( } } *smallerPtr = *largerPtr = 0; - if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ - assert(matchEndIdx > current + 8); - return matchEndIdx - (current + 8); + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > current + 8); + return MAX(positions, matchEndIdx - (current + 8)); + } } FORCE_INLINE_TEMPLATE @@ -520,8 +525,13 @@ void ZSTD_updateTree_internal( DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); - while(idx < target) - idx += ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); ms->nextToUpdate = target; } @@ -531,16 +541,18 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], + const U32 rep[ZSTD_REP_NUM], U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -556,8 +568,9 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 const dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = btMask >= current ? 0 : current - btMask; - U32 const windowLow = ms->window.lowLimit; + U32 const btLow = (btMask >= current) ? 0 : current - btMask; + U32 const windowValid = ms->window.lowLimit; + U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid; U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -627,7 +640,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( /* HC3 match finder */ if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip); + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); if ((matchIndex3 >= matchLow) & (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { size_t mlen; @@ -653,9 +666,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( (ip+mlen == iLimit) ) { /* best possible length */ ms->nextToUpdate = current+1; /* skip insertion */ return 1; - } - } - } + } } } /* no dictMatchState lookup: dicts don't have a populated HC3 table */ } @@ -760,10 +771,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( + ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */ ZSTD_matchState_t* ms, + U32* nextToUpdate3, const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], U32 const ll0, - ZSTD_match_t* matches, U32 const lengthToBeat) + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const matchLengthSearch = cParams->minMatch; @@ -772,12 +786,12 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); switch(matchLengthSearch) { - case 3 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3); + case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3); default : - case 4 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4); - case 5 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5); + case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4); + case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5); case 7 : - case 6 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6); + case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6); } } @@ -853,6 +867,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; @@ -862,7 +877,6 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); assert(optLevel <= 2); - ms->nextToUpdate3 = ms->nextToUpdate; ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); ip += (ip==prefixStart); @@ -873,7 +887,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* find first match */ { U32 const litlen = (U32)(ip - anchor); U32 const ll0 = !litlen; - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch); if (!nbMatches) { ip++; continue; } /* initialize opt[0] */ @@ -970,7 +984,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; U32 const previousPrice = opt[cur].price; U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); - U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch); + U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch); U32 matchNb; if (!nbMatches) { DEBUGLOG(7, "rPos:%u : no match found", cur); @@ -1094,7 +1108,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } /* while (ip < ilimit) */ /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -1158,7 +1172,6 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ms->window.dictLimit += (U32)srcSize; ms->window.lowLimit = ms->window.dictLimit; ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; /* re-inforce weight of collected statistics */ ZSTD_upscaleStats(&ms->opt); diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 38fbb907685..9e537b88485 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -1129,9 +1129,14 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); toFlush = produced - flushed; - if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) { - /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */ + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { assert(jobPtr->consumed < jobPtr->src.size); } } @@ -1148,12 +1153,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) { - if (params.ldmParams.enableLdm) + unsigned jobLog; + if (params.ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - return MAX(21, params.cParams.chainLog + 4); - return MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(21, params.cParams.chainLog + 4); + } else { + jobLog = MAX(20, params.cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) @@ -1197,7 +1206,7 @@ static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } - assert(0 <= ovLog && ovLog <= 30); + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); DEBUGLOG(4, "overlapLog : %i", params.overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; @@ -1391,7 +1400,7 @@ size_t ZSTDMT_initCStream_internal( FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) ); if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; - if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { @@ -1432,6 +1441,8 @@ size_t ZSTDMT_initCStream_internal( if (mtctx->targetSectionSize == 0) { mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + if (params.rsyncable) { /* Aim for the targetsectionSize as the average job size. */ U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); diff --git a/thirdparty/zstd/compress/zstdmt_compress.h b/thirdparty/zstd/compress/zstdmt_compress.h index 12e6bcb3a33..12a526087db 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.h +++ b/thirdparty/zstd/compress/zstdmt_compress.h @@ -50,6 +50,7 @@ #ifndef ZSTDMT_JOBSIZE_MIN # define ZSTDMT_JOBSIZE_MIN (1 MB) #endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) #define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index 675596f5aa9..e42872ad96a 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -360,8 +360,11 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported); - - return skippableHeaderSize + sizeU32; + { + size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong); + return skippableSize; + } } /** ZSTD_findDecompressedSize() : @@ -378,11 +381,10 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - if (srcSize < skippableSize) { + if (ZSTD_isError(skippableSize)) { return ZSTD_CONTENTSIZE_ERROR; } + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; @@ -467,6 +469,8 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); return frameSizeInfo; } else { const BYTE* ip = (const BYTE*)src; @@ -529,7 +533,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) return frameSizeInfo.compressedSize; } - /** ZSTD_decompressBound() : * compatible with legacy mode * `src` must point to the start of a ZSTD frame or a skippeable frame @@ -546,6 +549,7 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); src = (const BYTE*)src + compressedSize; srcSize -= compressedSize; bound += decompressedBound; @@ -738,9 +742,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, (unsigned)magicNumber, ZSTD_MAGICNUMBER); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) - return skippableSize; - RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong); + FORWARD_IF_ERROR(skippableSize); + assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index a2a7eedcf21..24f4859c56c 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -505,7 +505,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, *nbSeqPtr = nbSeq; /* FSE table descriptors */ - RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong); /* minimum possible size: 1 byte for symbol encoding types */ { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); @@ -637,9 +637,10 @@ size_t ZSTD_execSequence(BYTE* op, if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -686,13 +687,13 @@ size_t ZSTD_execSequence(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -717,9 +718,11 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); /* copy Literals */ - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + else + ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -766,13 +769,13 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (oMatchEnd > oend-(16-MINMATCH)) { if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); + ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); match += oend_w - op; op = oend_w; } while (op < oMatchEnd) *op++ = *match++; } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ } return sequenceLength; } @@ -889,6 +892,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) } FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, @@ -918,6 +922,11 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); @@ -930,6 +939,7 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* check if reached exact end */ DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); RETURN_ERROR_IF(nbSeq, corruption_detected); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } } @@ -1131,6 +1141,7 @@ ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG static TARGET_ATTRIBUTE("bmi2") size_t +DONT_VECTORIZE ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* seqStart, size_t seqSize, int nbSeq, diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index 53470c18f36..a1910ee223e 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -71,7 +71,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -82,16 +82,16 @@ ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library v #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ -/*************************************** -* Default constant -***************************************/ +/* ************************************* + * Default constant + ***************************************/ #ifndef ZSTD_CLEVEL_DEFAULT # define ZSTD_CLEVEL_DEFAULT 3 #endif -/*************************************** -* Constants -***************************************/ +/* ************************************* + * Constants + ***************************************/ /* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ @@ -183,9 +183,14 @@ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compres ***************************************/ /*= Compression context * When compressing many times, - * it is recommended to allocate a context just once, and re-use it for each successive compression operation. + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. * This will make workload friendlier for system's memory. - * Use one context per thread for parallel execution in multi-threaded environments. */ + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); @@ -380,6 +385,7 @@ typedef enum { * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -389,6 +395,7 @@ typedef enum { ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, } ZSTD_cParameter; typedef struct { @@ -657,17 +664,33 @@ ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_inBuffer* input, ZSTD_EndDirective endOp); -ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ -/******************************************************************************* - * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and - * ZSTD_compressStream2(). It is redundant, but is still fully supported. +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API. + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. * Advanced parameters and dictionary compression can only be used through the * new API. ******************************************************************************/ -/** +/*! * Equivalent to: * * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); @@ -675,16 +698,16 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); -/** +/*! * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). * NOTE: The return value is different. ZSTD_compressStream() returns a hint for * the next read size (if non-zero and not an error). ZSTD_compressStream2() - * returns the number of bytes left to flush (if non-zero and not an error). + * returns the minimum nb of bytes left to flush (if non-zero and not an error). */ ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); -/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); @@ -969,7 +992,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #endif /* ZSTD_H_235446 */ -/**************************************************************************************** +/* ************************************************************************************** * ADVANCED AND EXPERIMENTAL FUNCTIONS **************************************************************************************** * The definitions in the following section are considered experimental. @@ -1037,6 +1060,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_LDM_HASHRATELOG_MIN 0 #define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) +/* Advanced parameter bounds */ +#define ZSTD_TARGETCBLOCKSIZE_MIN 64 +#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX + /* internal */ #define ZSTD_HASHLOG3_MAX 17 @@ -1162,7 +1189,7 @@ typedef enum { * however it does mean that all frame data must be present and valid. */ ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); -/** ZSTD_decompressBound() : +/*! ZSTD_decompressBound() : * `src` should point to the start of a series of ZSTD encoded and/or skippable frames * `srcSize` must be the _exact_ size of this series * (i.e. there should be a frame boundary at `src + srcSize`) @@ -1409,6 +1436,11 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre */ #define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1843,7 +1875,7 @@ typedef struct { unsigned checksumFlag; } ZSTD_frameHeader; -/** ZSTD_getFrameHeader() : +/*! ZSTD_getFrameHeader() : * decode Frame Header, or requires larger `srcSize`. * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, From c5638393551be0f39cc6cb751554cb8b41a72f0d Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 1 Nov 2019 21:36:06 -0500 Subject: [PATCH 010/113] Update zstd to 1.4.3 (cherry picked from commit 61ad365fc9a1c20505706f02c0d713f07aff18be) --- core/SCsub | 2 + thirdparty/README.md | 2 +- thirdparty/zstd/common/bitstream.h | 4 + thirdparty/zstd/common/compiler.h | 6 +- thirdparty/zstd/common/mem.h | 2 +- thirdparty/zstd/common/xxhash.c | 10 +- thirdparty/zstd/common/zstd_internal.h | 8 +- thirdparty/zstd/compress/zstd_compress.c | 603 +----------------- .../zstd/compress/zstd_compress_internal.h | 82 ++- .../zstd/compress/zstd_compress_literals.c | 149 +++++ .../zstd/compress/zstd_compress_literals.h | 29 + .../zstd/compress/zstd_compress_sequences.c | 415 ++++++++++++ .../zstd/compress/zstd_compress_sequences.h | 47 ++ thirdparty/zstd/compress/zstd_double_fast.c | 5 +- thirdparty/zstd/compress/zstd_fast.c | 18 +- thirdparty/zstd/compress/zstd_lazy.c | 56 +- thirdparty/zstd/compress/zstd_opt.c | 10 +- thirdparty/zstd/decompress/zstd_decompress.c | 5 +- .../zstd/decompress/zstd_decompress_block.c | 7 +- thirdparty/zstd/zstd.h | 18 +- 20 files changed, 815 insertions(+), 663 deletions(-) create mode 100644 thirdparty/zstd/compress/zstd_compress_literals.c create mode 100644 thirdparty/zstd/compress/zstd_compress_literals.h create mode 100644 thirdparty/zstd/compress/zstd_compress_sequences.c create mode 100644 thirdparty/zstd/compress/zstd_compress_sequences.h diff --git a/core/SCsub b/core/SCsub index 00d0bcac24a..044939c3d76 100644 --- a/core/SCsub +++ b/core/SCsub @@ -121,6 +121,8 @@ if env['builtin_zstd']: "compress/zstd_ldm.c", "compress/zstd_opt.c", "compress/zstdmt_compress.c", + "compress/zstd_compress_literals.c", + "compress/zstd_compress_sequences.c", "decompress/huf_decompress.c", "decompress/zstd_ddict.c", "decompress/zstd_decompress_block.c", diff --git a/thirdparty/README.md b/thirdparty/README.md index 54c207b2a63..c0a6e6b3b2b 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -537,7 +537,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.1 +- Version: 1.4.3 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/zstd/common/bitstream.h b/thirdparty/zstd/common/bitstream.h index d955bd677b5..7bdb060460d 100644 --- a/thirdparty/zstd/common/bitstream.h +++ b/thirdparty/zstd/common/bitstream.h @@ -57,6 +57,8 @@ extern "C" { =========================================*/ #if defined(__BMI__) && defined(__GNUC__) # include /* support for bextr (experimental) */ +#elif defined(__ICCARM__) +# include #endif #define STREAM_ACCUMULATOR_MIN_32 25 @@ -163,6 +165,8 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ return 31 - __builtin_clz (val); +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 87bf51ae8c8..6686b837d64 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -23,7 +23,7 @@ # define INLINE_KEYWORD #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -65,7 +65,7 @@ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# ifdef __GNUC__ +# if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -76,7 +76,7 @@ #ifndef __has_attribute #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) diff --git a/thirdparty/zstd/common/mem.h b/thirdparty/zstd/common/mem.h index 5da248756ff..c10d7f61e1e 100644 --- a/thirdparty/zstd/common/mem.h +++ b/thirdparty/zstd/common/mem.h @@ -102,7 +102,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif diff --git a/thirdparty/zstd/common/xxhash.c b/thirdparty/zstd/common/xxhash.c index 30599aaae41..99d24596218 100644 --- a/thirdparty/zstd/common/xxhash.c +++ b/thirdparty/zstd/common/xxhash.c @@ -53,7 +53,8 @@ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ + defined(__ICCARM__) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -120,7 +121,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp # define INLINE_KEYWORD #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -206,7 +207,12 @@ static U64 XXH_read64(const void* memPtr) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) #else +#if defined(__ICCARM__) +# include +# define XXH_rotl32(x,r) __ROR(x,(32 - r)) +#else # define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 81b16eac2ea..585fd6b19ec 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -56,9 +56,9 @@ extern "C" { /** * Return the specified error if the condition evaluates to true. * - * In debug modes, prints additional information. In order to do that - * (particularly, printing the conditional that failed), this can't just wrap - * RETURN_ERROR(). + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). */ #define RETURN_ERROR_IF(cond, err, ...) \ if (cond) { \ @@ -324,6 +324,8 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus return (unsigned)r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ return 31 - __builtin_clz(val); +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); # else /* Software version */ static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 1476512580b..cd73db13be0 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -21,6 +21,8 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" #include "zstd_fast.h" #include "zstd_double_fast.h" #include "zstd_lazy.h" @@ -397,18 +399,6 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) } } -/* ZSTD_cParam_withinBounds: - * @return 1 if value is within cParam bounds, - * 0 otherwise */ -static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) -{ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); - if (ZSTD_isError(bounds.error)) return 0; - if (value < bounds.lowerBound) return 0; - if (value > bounds.upperBound) return 0; - return 1; -} - /* ZSTD_cParam_clampBounds: * Clamps the value into the bounded range. */ @@ -1903,155 +1893,6 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* s return ZSTD_blockHeaderSize + srcSize; } -static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - BYTE* const ostart = (BYTE* const)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - - RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); - - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); - } - - memcpy(ostart + flSize, src, srcSize); - return srcSize + flSize; -} - -static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - BYTE* const ostart = (BYTE* const)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ - - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); - } - - ostart[flSize] = *(const BYTE*)src; - return flSize+1; -} - - -/* ZSTD_minGain() : - * minimum compression required - * to generate a compress block or a compressed literals section. - * note : use same formula for both situations */ -static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) -{ - U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; - ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); - return (srcSize >> minlog) + 2; -} - -static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* workspace, size_t wkspSize, - const int bmi2) -{ - size_t const minGain = ZSTD_minGain(srcSize, strategy); - size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); - BYTE* const ostart = (BYTE*)dst; - U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; - size_t cLitSize; - - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)", - disableLiteralCompression); - - /* Prepare nextEntropy assuming reusing the existing table */ - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - - if (disableLiteralCompression) - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - - RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); - { HUF_repeat repeat = prevHuf->repeatMode; - int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; - if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); - if (repeat != HUF_repeat_none) { - /* reused the existing table */ - hType = set_repeat; - } - } - - if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - if (cLitSize==1) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } - - if (hType == set_compressed) { - /* using a newly constructed table */ - nextHuf->repeatMode = HUF_repeat_check; - } - - /* Build header */ - switch(lhSize) - { - case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); - MEM_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); - MEM_writeLE32(ostart, lhc); - break; - } - case 5: /* 2 - 2 - 18 - 18 */ - { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); - MEM_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - default: /* not possible : lhSize is {3,4,5} */ - assert(0); - } - return lhSize+cLitSize; -} - - void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { const seqDef* const sequences = seqStorePtr->sequencesStart; @@ -2074,418 +1915,6 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; } - -/** - * -log2(x / 256) lookup table for x in [0, 256). - * If x == 0: Return 0 - * Else: Return floor(-log2(x / 256) * 256) - */ -static unsigned const kInverseProbabilityLog256[256] = { - 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, - 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, - 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, - 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, - 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, - 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, - 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, - 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, - 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, - 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, - 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, - 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, - 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, - 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, - 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, - 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, - 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, - 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, - 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, - 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, - 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, - 5, 4, 2, 1, -}; - - -/** - * Returns the cost in bits of encoding the distribution described by count - * using the entropy bound. - */ -static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) -{ - unsigned cost = 0; - unsigned s; - for (s = 0; s <= max; ++s) { - unsigned norm = (unsigned)((256 * count[s]) / total); - if (count[s] != 0 && norm == 0) - norm = 1; - assert(count[s] < total); - cost += count[s] * kInverseProbabilityLog256[norm]; - } - return cost >> 8; -} - - -/** - * Returns the cost in bits of encoding the distribution in count using the - * table described by norm. The max symbol support by norm is assumed >= max. - * norm must be valid for every symbol with non-zero probability in count. - */ -static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max) -{ - unsigned const shift = 8 - accuracyLog; - size_t cost = 0; - unsigned s; - assert(accuracyLog <= 8); - for (s = 0; s <= max; ++s) { - unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; - unsigned const norm256 = normAcc << shift; - assert(norm256 > 0); - assert(norm256 < 256); - cost += count[s] * kInverseProbabilityLog256[norm256]; - } - return cost >> 8; -} - - -static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { - void const* ptr = ctable; - U16 const* u16ptr = (U16 const*)ptr; - U32 const maxSymbolValue = MEM_read16(u16ptr + 1); - return maxSymbolValue; -} - - -/** - * Returns the cost in bits of encoding the distribution in count using ctable. - * Returns an error if ctable cannot represent all the symbols in count. - */ -static size_t ZSTD_fseBitCost( - FSE_CTable const* ctable, - unsigned const* count, - unsigned const max) -{ - unsigned const kAccuracyLog = 8; - size_t cost = 0; - unsigned s; - FSE_CState_t cstate; - FSE_initCState(&cstate, ctable); - RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, - "Repeat FSE_CTable has maxSymbolValue %u < %u", - ZSTD_getFSEMaxSymbolValue(ctable), max); - for (s = 0; s <= max; ++s) { - unsigned const tableLog = cstate.stateLog; - unsigned const badCost = (tableLog + 1) << kAccuracyLog; - unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); - if (count[s] == 0) - continue; - RETURN_ERROR_IF(bitCost >= badCost, GENERIC, - "Repeat FSE_CTable has Prob[%u] == 0", s); - cost += count[s] * bitCost; - } - return cost >> kAccuracyLog; -} - -/** - * Returns the cost in bytes of encoding the normalized count header. - * Returns an error if any of the helper functions return an error. - */ -static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, - size_t const nbSeq, unsigned const FSELog) -{ - BYTE wksp[FSE_NCOUNTBOUND]; - S16 norm[MaxSeq + 1]; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); - return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); -} - - -typedef enum { - ZSTD_defaultDisallowed = 0, - ZSTD_defaultAllowed = 1 -} ZSTD_defaultPolicy_e; - -MEM_STATIC symbolEncodingType_e -ZSTD_selectEncodingType( - FSE_repeat* repeatMode, unsigned const* count, unsigned const max, - size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, - FSE_CTable const* prevCTable, - short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, - ZSTD_strategy const strategy) -{ - ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); - if (mostFrequent == nbSeq) { - *repeatMode = FSE_repeat_none; - if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or less symbols, - * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. - * If basic encoding isn't possible, always choose RLE. - */ - DEBUGLOG(5, "Selected set_basic"); - return set_basic; - } - DEBUGLOG(5, "Selected set_rle"); - return set_rle; - } - if (strategy < ZSTD_lazy) { - if (isDefaultAllowed) { - size_t const staticFse_nbSeq_max = 1000; - size_t const mult = 10 - strategy; - size_t const baseLog = 3; - size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ - assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ - assert(mult <= 9 && mult >= 7); - if ( (*repeatMode == FSE_repeat_valid) - && (nbSeq < staticFse_nbSeq_max) ) { - DEBUGLOG(5, "Selected set_repeat"); - return set_repeat; - } - if ( (nbSeq < dynamicFse_nbSeq_min) - || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { - DEBUGLOG(5, "Selected set_basic"); - /* The format allows default tables to be repeated, but it isn't useful. - * When using simple heuristics to select encoding type, we don't want - * to confuse these tables with dictionaries. When running more careful - * analysis, we don't need to waste time checking both repeating tables - * and default tables. - */ - *repeatMode = FSE_repeat_none; - return set_basic; - } - } - } else { - size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); - size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); - size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); - size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); - - if (isDefaultAllowed) { - assert(!ZSTD_isError(basicCost)); - assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); - } - assert(!ZSTD_isError(NCountCost)); - assert(compressedCost < ERROR(maxCode)); - DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", - (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); - if (basicCost <= repeatCost && basicCost <= compressedCost) { - DEBUGLOG(5, "Selected set_basic"); - assert(isDefaultAllowed); - *repeatMode = FSE_repeat_none; - return set_basic; - } - if (repeatCost <= compressedCost) { - DEBUGLOG(5, "Selected set_repeat"); - assert(!ZSTD_isError(repeatCost)); - return set_repeat; - } - assert(compressedCost < basicCost && compressedCost < repeatCost); - } - DEBUGLOG(5, "Selected set_compressed"); - *repeatMode = FSE_repeat_check; - return set_compressed; -} - -MEM_STATIC size_t -ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, - unsigned* count, U32 max, - const BYTE* codeTable, size_t nbSeq, - const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, - const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize) -{ - BYTE* op = (BYTE*)dst; - const BYTE* const oend = op + dstCapacity; - DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); - - switch (type) { - case set_rle: - FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); - RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); - *op = codeTable[0]; - return 1; - case set_repeat: - memcpy(nextCTable, prevCTable, prevCTableSize); - return 0; - case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ - return 0; - case set_compressed: { - S16 norm[MaxSeq + 1]; - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - if (count[codeTable[nbSeq-1]] > 1) { - count[codeTable[nbSeq-1]]--; - nbSeq_1--; - } - assert(nbSeq_1 > 1); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); - { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); - return NCountSize; - } - } - default: assert(0); RETURN_ERROR(GENERIC); - } -} - -FORCE_INLINE_TEMPLATE size_t -ZSTD_encodeSequences_body( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - RETURN_ERROR_IF( - ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), - dstSize_tooSmall, "not enough space remaining"); - DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", - (int)(blockStream.endPtr - blockStream.startPtr), - (unsigned)dstCapacity); - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq-1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, - ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); - } - BIT_flushBits(&blockStream); - - { size_t n; - for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, - ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); - } } - - DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); - FSE_flushCState(&blockStream, &stateMatchLength); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); - FSE_flushCState(&blockStream, &stateOffsetBits); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); - FSE_flushCState(&blockStream, &stateLitLength); - - { size_t const streamSize = BIT_closeCStream(&blockStream); - RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); - return streamSize; - } -} - -static size_t -ZSTD_encodeSequences_default( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - - -#if DYNAMIC_BMI2 - -static TARGET_ATTRIBUTE("bmi2") size_t -ZSTD_encodeSequences_bmi2( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - -#endif - -static size_t ZSTD_encodeSequences( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) -{ - DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { - return ZSTD_encodeSequences_bmi2(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); - } -#endif - (void)bmi2; - return ZSTD_encodeSequences_default(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) { switch (cctxParams->literalCompressionMode) { @@ -2526,16 +1955,16 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); BYTE* seqHead; BYTE* lastNCount = NULL; + DEBUGLOG(5, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<litStart; - size_t const litSize = seqStorePtr->lit - literals; + size_t const litSize = (size_t)(seqStorePtr->lit - literals); size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, cctxParams->cParams.strategy, @@ -2562,7 +1991,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); - return op - ostart; + return (size_t)(op - ostart); } /* seqHead : flags for FSE encoding type */ @@ -2583,7 +2012,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), workspace, wkspSize); @@ -2606,7 +2035,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), workspace, wkspSize); @@ -2627,7 +2056,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), workspace, wkspSize); @@ -2641,7 +2070,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); { size_t const bitstreamSize = ZSTD_encodeSequences( - op, oend - op, + op, (size_t)(oend - op), CTable_MatchLength, mlCodeTable, CTable_OffsetBits, ofCodeTable, CTable_LitLength, llCodeTable, @@ -2668,7 +2097,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); - return op - ostart; + return (size_t)(op - ostart); } MEM_STATIC size_t @@ -2841,7 +2270,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { size_t cSize; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss); @@ -3109,8 +2539,9 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t const blockSizeMax = ZSTD_getBlockSize(cctx); - RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } @@ -3135,7 +2566,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, if (srcSize <= HASH_READ_SIZE) return 0; while (iend - ip > HASH_READ_SIZE) { - size_t const remaining = iend - ip; + size_t const remaining = (size_t)(iend - ip); size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); const BYTE* const ichunk = ip + chunk; diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index 5495899be35..6d623cc6be8 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -134,9 +134,15 @@ typedef struct { typedef struct ZSTD_matchState_t ZSTD_matchState_t; struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ U32 nextToUpdate; /* index from which to continue table update */ - U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ U32* hashTable; U32* hashTable3; U32* chainTable; @@ -307,6 +313,30 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; } +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + return (srcSize >> minlog) + 2; +} + /*! ZSTD_storeSeq() : * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). @@ -326,7 +356,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); + ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); seqStorePtr->lit += litLength; /* literal Length */ @@ -739,24 +769,37 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, /* Similar to ZSTD_window_enforceMaxDist(), * but only invalidates dictionary - * when input progresses beyond window size. */ + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ MEM_STATIC void -ZSTD_checkDictValidity(ZSTD_window_t* window, +ZSTD_checkDictValidity(const ZSTD_window_t* window, const void* blockEnd, U32 maxDist, U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); - U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", - (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); - if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) { - /* On reaching window size, dictionaries are invalidated */ - if (loadedDictEndPtr) *loadedDictEndPtr = 0; - if (dictMatchStatePtr) *dictMatchStatePtr = NULL; - } + if (blockEndIdx > loadedDictEnd + maxDist) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } } /** @@ -798,6 +841,17 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, return contiguous; } +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + /* debug functions */ #if (DEBUGLEVEL>=2) diff --git a/thirdparty/zstd/compress/zstd_compress_literals.c b/thirdparty/zstd/compress/zstd_compress_literals.c new file mode 100644 index 00000000000..eb3e5a44bc0 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_literals.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + return flSize+1; +} + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + const int bmi2) +{ + size_t const minGain = ZSTD_minGain(srcSize, strategy); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)", + disableLiteralCompression); + + /* Prepare nextEntropy assuming reusing the existing table */ + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) + : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + hType = set_repeat; + } + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize==1) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + return lhSize+cLitSize; +} diff --git a/thirdparty/zstd/compress/zstd_compress_literals.h b/thirdparty/zstd/compress/zstd_compress_literals.h new file mode 100644 index 00000000000..7adbecc0beb --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_literals.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + const int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.c b/thirdparty/zstd/compress/zstd_compress_sequences.c new file mode 100644 index 00000000000..3c3deae08cb --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_sequences.c @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +static size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, + "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + RETURN_ERROR_IF(bitCost >= badCost, GENERIC, + "Repeat FSE_CTable has Prob[%u] == 0", s); + cost += count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* workspace, size_t workspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); + *op = codeTable[0]; + return 1; + case set_repeat: + memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + S16 norm[MaxSeq + 1]; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.h b/thirdparty/zstd/compress/zstd_compress_sequences.h new file mode 100644 index 00000000000..f5234d94c81 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_sequences.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "fse.h" /* FSE_repeat, FSE_CTable */ +#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* workspace, size_t workspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 5957255d904..54467cc31bd 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -65,6 +65,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowestValid = ms->window.dictLimit; const U32 maxDistance = 1U << cParams->windowLog; + /* presumes that, if there is a dictionary, it must be using Attach mode */ const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; @@ -369,9 +370,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const ilimit = iend - 8; const BYTE* const base = ms->window.base; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 lowestValid = ms->window.lowLimit; - const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const U32 dictLimit = ms->window.dictLimit; const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index a05b8a47f14..59267ffbbcf 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -71,6 +71,7 @@ size_t ZSTD_compressBlock_fast_generic( U32 offsetSaved = 0; /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; { @@ -239,6 +240,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); ip += (dictAndPrefixLength == 0); /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ @@ -379,9 +381,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 validLow = ms->window.lowLimit; - const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; const U32 dictLimit = ms->window.dictLimit; @@ -392,6 +392,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic"); + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); @@ -412,8 +414,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; ip++; ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); } else { @@ -423,8 +425,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( ip += ((ip-anchor) >> kSearchStrength) + stepSize; continue; } - { const BYTE* matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; U32 offset; mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ @@ -451,7 +453,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 94d906c01f1..0af41724c7c 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -242,9 +242,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const maxDistance = 1U << cParams->windowLog; - U32 const windowValid = ms->window.lowLimit; - U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -497,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic ( const BYTE* const dictEnd = dictBase + dictLimit; const U32 current = (U32)(ip-base); const U32 maxDistance = 1U << cParams->windowLog; - const U32 lowValid = ms->window.lowLimit; - const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; const U32 minChain = current > chainSize ? current - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; @@ -619,12 +619,14 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( /* ******************************* * Common parser - lazy strategy *********************************/ -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_lazy_generic( +typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth, + const searchMethod_e searchMethod, const U32 depth, ZSTD_dictMode_e const dictMode) { const BYTE* const istart = (const BYTE*)src; @@ -640,8 +642,10 @@ size_t ZSTD_compressBlock_lazy_generic( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ? - (searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : - (searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS); + (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS + : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : + (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_selectMLS + : ZSTD_HcFindBestMatch_selectMLS); U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; const ZSTD_matchState_t* const dms = ms->dictMatchState; @@ -850,7 +854,7 @@ _storeSequence: rep[1] = offset_2 ? offset_2 : savedOffset; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -858,56 +862,56 @@ size_t ZSTD_compressBlock_btlazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); } size_t ZSTD_compressBlock_greedy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); } size_t ZSTD_compressBlock_btlazy2_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_lazy2_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_lazy_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_greedy_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); } @@ -916,7 +920,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth) + const searchMethod_e searchMethod, const U32 depth) { const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; @@ -934,7 +938,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1]; @@ -1075,7 +1079,7 @@ _storeSequence: rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -1083,7 +1087,7 @@ size_t ZSTD_compressBlock_greedy_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 0); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); } size_t ZSTD_compressBlock_lazy_extDict( @@ -1091,7 +1095,7 @@ size_t ZSTD_compressBlock_lazy_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 1); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); } size_t ZSTD_compressBlock_lazy2_extDict( @@ -1099,7 +1103,7 @@ size_t ZSTD_compressBlock_lazy2_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); } size_t ZSTD_compressBlock_btlazy2_extDict( @@ -1107,5 +1111,5 @@ size_t ZSTD_compressBlock_btlazy2_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 1, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); } diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index e32e542e021..2da363f93ef 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -552,7 +552,6 @@ U32 ZSTD_insertBtAndGetAllMatches ( { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -569,8 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32 const btLow = (btMask >= current) ? 0 : current - btMask; - U32 const windowValid = ms->window.lowLimit; - U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -674,19 +672,21 @@ U32 ZSTD_insertBtAndGetAllMatches ( while (nbCompares-- && (matchIndex >= matchLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(current > matchIndex); if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); } else { match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* prepare for match[matchLength] */ + match = base + matchIndex; /* prepare for match[matchLength] read */ } if (matchLength > bestLength) { diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index e42872ad96a..751060b2cd1 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -574,9 +574,10 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) } /** ZSTD_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) { + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); ZSTD_checkContinuity(dctx, blockStart); dctx->previousDstEnd = (const char*)blockStart + blockSize; return blockSize; @@ -909,6 +910,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); dctx->expected = cBlockSize; dctx->bType = bp.blockType; dctx->rleSize = bp.origSize; @@ -953,6 +955,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c RETURN_ERROR(corruption_detected); } if (ZSTD_isError(rSize)) return rSize; + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); dctx->decodedSize += rSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index 24f4859c56c..cbcfc084061 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -79,6 +79,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ { + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected); { const BYTE* const istart = (const BYTE*) src; @@ -87,6 +88,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, switch(litEncType) { case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted); /* fall-through */ @@ -116,7 +118,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, /* 2 - 2 - 18 - 18 */ lhSize = 5; litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); @@ -391,7 +393,8 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt, symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; } } } memcpy(dt, &DTableH, sizeof(DTableH)); } diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index a1910ee223e..f8e95f2283e 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -71,7 +71,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 3 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -1909,7 +1909,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); /*! Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). - User will have to take in charge required information to regenerate data, such as compressed and content sizes. + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. A few rules to respect : - Compressing and decompressing require a context structure @@ -1920,12 +1920,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, really consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - In which case, nothing is produced into `dst` ! - + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + In case of multiple successive blocks, should some of them be uncompressed, decoder must be informed of their existence in order to follow proper history. Use ZSTD_insertBlock() for such a case. From bc31b11fdd9af57e7f8c45b6e1a819f01306e9d2 Mon Sep 17 00:00:00 2001 From: Jonathan Mannancheril Date: Sat, 9 Nov 2019 22:31:00 -0600 Subject: [PATCH 011/113] Update zstd to 1.4.4 (cherry picked from commit 55afd6e784d8dea6779f471d35a77e4d56bfaaca) --- thirdparty/README.md | 2 +- thirdparty/zstd/common/bitstream.h | 5 +- thirdparty/zstd/common/compiler.h | 16 +- thirdparty/zstd/common/fse.h | 2 +- thirdparty/zstd/common/fse_decompress.c | 2 + thirdparty/zstd/common/mem.h | 73 ++ thirdparty/zstd/common/pool.c | 10 +- thirdparty/zstd/common/threading.c | 47 +- thirdparty/zstd/common/threading.h | 33 +- thirdparty/zstd/common/zstd_internal.h | 87 +- thirdparty/zstd/compress/zstd_compress.c | 1077 ++++++++++------- .../zstd/compress/zstd_compress_internal.h | 66 +- .../zstd/compress/zstd_compress_literals.c | 15 +- .../zstd/compress/zstd_compress_literals.h | 2 +- .../zstd/compress/zstd_compress_sequences.c | 6 +- .../zstd/compress/zstd_compress_sequences.h | 2 +- thirdparty/zstd/compress/zstd_cwksp.h | 535 ++++++++ thirdparty/zstd/compress/zstd_double_fast.c | 18 +- thirdparty/zstd/compress/zstd_fast.c | 69 +- thirdparty/zstd/compress/zstd_lazy.c | 10 +- thirdparty/zstd/compress/zstd_ldm.c | 8 +- thirdparty/zstd/compress/zstd_opt.c | 2 +- thirdparty/zstd/compress/zstdmt_compress.c | 58 +- thirdparty/zstd/decompress/huf_decompress.c | 2 + thirdparty/zstd/decompress/zstd_decompress.c | 30 +- .../zstd/decompress/zstd_decompress_block.c | 286 +++-- thirdparty/zstd/zstd.h | 220 +++- 27 files changed, 1852 insertions(+), 831 deletions(-) create mode 100644 thirdparty/zstd/compress/zstd_cwksp.h diff --git a/thirdparty/README.md b/thirdparty/README.md index c0a6e6b3b2b..ff6156d42e7 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -537,7 +537,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.3 +- Version: 1.4.4 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/zstd/common/bitstream.h b/thirdparty/zstd/common/bitstream.h index 7bdb060460d..1c294b80d13 100644 --- a/thirdparty/zstd/common/bitstream.h +++ b/thirdparty/zstd/common/bitstream.h @@ -164,7 +164,7 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ return 31 - __CLZ(val); # else /* Software version */ @@ -244,9 +244,9 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; - assert(bitC->ptr <= bitC->endPtr); bitC->bitPos &= 7; bitC->bitContainer >>= nbBytes*8; } @@ -260,6 +260,7 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 6686b837d64..1877a0c1d9b 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -61,6 +61,13 @@ # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) @@ -127,9 +134,14 @@ } \ } -/* vectorization */ +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ #if !defined(__clang__) && defined(__GNUC__) -# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif #else # define DONT_VECTORIZE #endif diff --git a/thirdparty/zstd/common/fse.h b/thirdparty/zstd/common/fse.h index 811c670bddc..a7553e3721c 100644 --- a/thirdparty/zstd/common/fse.h +++ b/thirdparty/zstd/common/fse.h @@ -308,7 +308,7 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ diff --git a/thirdparty/zstd/common/fse_decompress.c b/thirdparty/zstd/common/fse_decompress.c index 72bbead5bee..4f073789820 100644 --- a/thirdparty/zstd/common/fse_decompress.c +++ b/thirdparty/zstd/common/fse_decompress.c @@ -52,7 +52,9 @@ #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ /* check and forward error code */ +#ifndef CHECK_F #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } +#endif /* ************************************************************** diff --git a/thirdparty/zstd/common/mem.h b/thirdparty/zstd/common/mem.h index c10d7f61e1e..530d30c8f75 100644 --- a/thirdparty/zstd/common/mem.h +++ b/thirdparty/zstd/common/mem.h @@ -47,6 +47,79 @@ extern "C" { #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } +/* detects whether we are being compiled under msan */ +#if defined (__has_feature) +# if __has_feature(memory_sanitizer) +# define MEMORY_SANITIZER 1 +# endif +#endif + +#if defined (MEMORY_SANITIZER) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +#include /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); +#endif + +/* detects whether we are being compiled under asan */ +#if defined (__has_feature) +# if __has_feature(address_sanitizer) +# define ADDRESS_SANITIZER 1 +# endif +#elif defined(__SANITIZE_ADDRESS__) +# define ADDRESS_SANITIZER 1 +#endif + +#if defined (ADDRESS_SANITIZER) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + /*-************************************************************** * Basic Types diff --git a/thirdparty/zstd/common/pool.c b/thirdparty/zstd/common/pool.c index 7a829454328..f575935076c 100644 --- a/thirdparty/zstd/common/pool.c +++ b/thirdparty/zstd/common/pool.c @@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ctx->queueTail = 0; ctx->numThreadsBusy = 0; ctx->queueEmpty = 1; - (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); - (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); - (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } ctx->shutdown = 0; /* Allocate space for the thread handles */ ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); diff --git a/thirdparty/zstd/common/threading.c b/thirdparty/zstd/common/threading.c index f3d4fa84184..482664bd9ad 100644 --- a/thirdparty/zstd/common/threading.c +++ b/thirdparty/zstd/common/threading.c @@ -14,6 +14,8 @@ * This file will hold wrapper for systems, which do not support pthreads */ +#include "threading.h" + /* create fake symbol to avoid empty translation unit warning */ int g_ZSTD_threading_useless_symbol; @@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol; /* === Dependencies === */ #include #include -#include "threading.h" /* === Implementation === */ @@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) } #endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#include + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + free(*cond); + return ret; + } +} + +#endif diff --git a/thirdparty/zstd/common/threading.h b/thirdparty/zstd/common/threading.h index d806c89d01c..3193ca7db86 100644 --- a/thirdparty/zstd/common/threading.h +++ b/thirdparty/zstd/common/threading.h @@ -13,6 +13,8 @@ #ifndef THREADING_H_938743 #define THREADING_H_938743 +#include "debug.h" + #if defined (__cplusplus) extern "C" { #endif @@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); */ -#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ /* === POSIX Systems === */ # include +#if DEBUGLEVEL < 1 + #define ZSTD_pthread_mutex_t pthread_mutex_t #define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) #define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) @@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) #define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) + +#endif + #else /* ZSTD_MULTITHREAD not defined */ /* No multithreading support */ diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 585fd6b19ec..dcdcbdb81cd 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -197,79 +197,56 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } -#define WILDCOPY_OVERLENGTH 8 -#define VECLEN 16 +#define WILDCOPY_OVERLENGTH 32 +#define WILDCOPY_VECLEN 16 typedef enum { ZSTD_no_overlap, - ZSTD_overlap_src_before_dst, + ZSTD_overlap_src_before_dst /* ZSTD_overlap_dst_before_src, */ } ZSTD_overlap_e; /*! ZSTD_wildcopy() : - * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ + * Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); - if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { - do - COPY8(op, ip) - while (op < oend); - } - else { - if ((length & 8) == 0) - COPY8(op, ip); - do { + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip) + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first two COPY16() calls because the copy length is + * almost certain to be short, so the branches have different + * probabilities. + * On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%. + * On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%. + */ COPY16(op, ip); - } - while (op < oend); - } -} - -/*! ZSTD_wildcopy_16min() : - * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ -MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) -{ - ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - - assert(length >= 8); - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); - - if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { - do - COPY8(op, ip) - while (op < oend); - } - else { - if ((length & 8) == 0) - COPY8(op, ip); - do { COPY16(op, ip); - } - while (op < oend); + if (op >= oend) return; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); } } -MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = (BYTE*)dstEnd; - do - COPY8(op, ip) - while (op < oend); -} - /*-******************************************* * Private declarations @@ -323,7 +300,7 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus _BitScanReverse(&r, val); return (unsigned)r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); + return __builtin_clz (val) ^ 31; # elif defined(__ICCARM__) /* IAR Intrinsic */ return 31 - __CLZ(val); # else /* Software version */ diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index cd73db13be0..35346b92cb1 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -42,15 +42,15 @@ size_t ZSTD_compressBound(size_t srcSize) { * Context memory management ***************************************/ struct ZSTD_CDict_s { - void* dictBuffer; const void* dictContent; size_t dictContentSize; - void* workspace; - size_t workspaceSize; + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -84,23 +84,26 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) { + return NULL; + } + memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; - cctx->workSpace = (void*)(cctx+1); - cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workSpace; - cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; - { - void* const ptr = cctx->blockState.nextCBlock + 1; - cctx->entropyWorkspace = (U32*)ptr; - } + if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object( + &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -128,11 +131,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) @@ -140,8 +143,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - ZSTD_freeCCtxContent(cctx); - ZSTD_free(cctx, cctx->customMem); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } + } return 0; } @@ -160,7 +168,9 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workSpaceSize + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -229,23 +239,23 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete RETURN_ERROR_IF(!cctxParams, GENERIC); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); memset(cctxParams, 0, sizeof(*cctxParams)); + assert(!ZSTD_checkCParams(params.cParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ - assert(!ZSTD_checkCParams(params.cParams)); return 0; } /* ZSTD_assignParamsToCCtxParams() : * params is presumed valid at this stage */ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( - ZSTD_CCtx_params cctxParams, ZSTD_parameters params) + const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { - ZSTD_CCtx_params ret = cctxParams; + ZSTD_CCtx_params ret = *cctxParams; + assert(!ZSTD_checkCParams(params.cParams)); ret.cParams = params.cParams; ret.fParams = params.fParams; ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ - assert(!ZSTD_checkCParams(params.cParams)); return ret; } @@ -378,7 +388,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -392,6 +402,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; return bounds; + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -448,6 +463,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: default: return 0; } @@ -494,6 +510,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: break; default: RETURN_ERROR(parameter_unsupported); @@ -517,33 +534,33 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, if (value) { /* 0 : does not change current level */ CCtxParams->compressionLevel = value; } - if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; return 0; /* return type (size_t) cannot represent negative values */ } case ZSTD_c_windowLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_windowLog, value); - CCtxParams->cParams.windowLog = value; + CCtxParams->cParams.windowLog = (U32)value; return CCtxParams->cParams.windowLog; case ZSTD_c_hashLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_hashLog, value); - CCtxParams->cParams.hashLog = value; + CCtxParams->cParams.hashLog = (U32)value; return CCtxParams->cParams.hashLog; case ZSTD_c_chainLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_chainLog, value); - CCtxParams->cParams.chainLog = value; + CCtxParams->cParams.chainLog = (U32)value; return CCtxParams->cParams.chainLog; case ZSTD_c_searchLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_searchLog, value); - CCtxParams->cParams.searchLog = value; - return value; + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; case ZSTD_c_minMatch : if (value!=0) /* 0 => use default */ @@ -674,6 +691,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->targetCBlockSize = value; return CCtxParams->targetCBlockSize; + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return CCtxParams->srcSizeHint; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -779,6 +802,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_targetCBlockSize : *value = (int)CCtxParams->targetCBlockSize; break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1029,7 +1055,11 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) { - ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; @@ -1049,10 +1079,19 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; @@ -1069,20 +1108,23 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); - size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + matchStateSize + ldmSpace + ldmSeqSpace; + size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); - return sizeof(ZSTD_CCtx) + neededSpace; + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace); + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return cctxSpace + neededSpace; } } @@ -1118,7 +1160,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = inBuffSize + outBuffSize; + size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize) + + ZSTD_cwksp_alloc_size(outBuffSize); return CCtxSize + streamingSize; } @@ -1186,17 +1229,6 @@ size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ } - - -static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, - ZSTD_compressionParameters cParams2) -{ - return (cParams1.hashLog == cParams2.hashLog) - & (cParams1.chainLog == cParams2.chainLog) - & (cParams1.strategy == cParams2.strategy) /* opt parser space */ - & ((cParams1.minMatch==3) == (cParams2.minMatch==3)); /* hashlog3 space */ -} - static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) { @@ -1211,71 +1243,6 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, assert(cParams1.strategy == cParams2.strategy); } -/** The parameters are equivalent if ldm is not enabled in both sets or - * all the parameters are equivalent. */ -static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, - ldmParams_t ldmParams2) -{ - return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || - (ldmParams1.enableLdm == ldmParams2.enableLdm && - ldmParams1.hashLog == ldmParams2.hashLog && - ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && - ldmParams1.minMatchLength == ldmParams2.minMatchLength && - ldmParams1.hashRateLog == ldmParams2.hashRateLog); -} - -typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; - -/* ZSTD_sufficientBuff() : - * check internal buffers exist for streaming if buffPol == ZSTDb_buffered . - * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */ -static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, - size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - ZSTD_compressionParameters cParams2, - U64 pledgedSrcSize) -{ - size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); - size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); - size_t const maxNbSeq2 = blockSize2 / ((cParams2.minMatch == 3) ? 3 : 4); - size_t const maxNbLit2 = blockSize2; - size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; - DEBUGLOG(4, "ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u", - (U32)neededBufferSize2, (U32)bufferSize1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbSeq2=%u <= maxNbSeq1=%u", - (U32)maxNbSeq2, (U32)maxNbSeq1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbLit2=%u <= maxNbLit1=%u", - (U32)maxNbLit2, (U32)maxNbLit1); - return (maxNbLit2 <= maxNbLit1) - & (maxNbSeq2 <= maxNbSeq1) - & (neededBufferSize2 <= bufferSize1); -} - -/** Equivalence for resetCCtx purposes */ -static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, - ZSTD_CCtx_params params2, - size_t buffSize1, - size_t maxNbSeq1, size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - U64 pledgedSrcSize) -{ - DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); - if (!ZSTD_equivalentCParams(params1.cParams, params2.cParams)) { - DEBUGLOG(4, "ZSTD_equivalentCParams() == 0"); - return 0; - } - if (!ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams)) { - DEBUGLOG(4, "ZSTD_equivalentLdmParams() == 0"); - return 0; - } - if (!ZSTD_sufficientBuff(buffSize1, maxNbSeq1, maxNbLit1, buffPol2, - params2.cParams, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_sufficientBuff() == 0"); - return 0; - } - return 1; -} - static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { int i; @@ -1301,87 +1268,104 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) ms->dictMatchState = NULL; } -/*! ZSTD_continueCCtx() : - * reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) -{ - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); +/** + * Indicates whether this compression proceeds directly from user-provided + * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or + * whether the context needs to buffer the input/output (ZSTDb_buffered). + */ +typedef enum { + ZSTDb_not_buffered, + ZSTDb_buffered +} ZSTD_buffered_policy_e; - cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ - cctx->appliedParams = params; - cctx->blockState.matchState.cParams = params.cParams; - cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; - cctx->consumedSrcSize = 0; - cctx->producedCSize = 0; - if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) - cctx->appliedParams.fParams.contentSizeFlag = 0; - DEBUGLOG(4, "pledged content size : %u ; flag : %u", - (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); - cctx->stage = ZSTDcs_init; - cctx->dictID = 0; - if (params.ldmParams.enableLdm) - ZSTD_window_clear(&cctx->ldmState.window); - ZSTD_referenceExternalSequences(cctx, NULL, 0); - ZSTD_invalidateMatchState(&cctx->blockState.matchState); - ZSTD_reset_compressedBlockState(cctx->blockState.prevCBlock); - XXH64_reset(&cctx->xxhState, 0); - return 0; -} +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; -typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; -static void* +static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - void* ptr, + ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - assert(((size_t)ptr & 3) == 0); + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + if (forceResetIndex == ZSTDirp_reset) { + memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_cwksp_mark_tables_dirty(ws); + } ms->hashLog3 = hashLog3; - memset(&ms->window, 0, sizeof(ms->window)); - ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ - ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_invalidateMatchState(ms); + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ptr; - ms->opt.litLengthFreq = ms->opt.litFreq + (1<opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL+1); - ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML+1); - ptr = ms->opt.offCodeFreq + (MaxOff+1); - ms->opt.matchTable = (ZSTD_match_t*)ptr; - ptr = ms->opt.matchTable + ZSTD_OPT_NUM+1; - ms->opt.priceTable = (ZSTD_optimal_t*)ptr; - ptr = ms->opt.priceTable + ZSTD_OPT_NUM+1; + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } - /* table Space */ - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ - ms->hashTable = (U32*)(ptr); - ms->chainTable = ms->hashTable + hSize; - ms->hashTable3 = ms->chainTable + chainSize; - ptr = ms->hashTable3 + h3Size; - ms->cParams = *cParams; - assert(((size_t)ptr & 3) == 0); - return ptr; + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + return 0; } /* ZSTD_indexTooCloseToMax() : @@ -1397,13 +1381,6 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, @@ -1412,30 +1389,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { + ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - if (crp == ZSTDcrp_continue) { - if (ZSTD_equivalentParams(zc->appliedParams, params, - zc->inBuffSize, - zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize) ) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { - DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); - if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { - /* prefer a reset, faster than a rescale */ - ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); - } - return ZSTD_continueCCtx(zc, params, pledgedSrcSize); - } } } - DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); + zc->isFirstBlock = 1; if (params.ldmParams.enableLdm) { /* Adjust long distance matching parameters */ @@ -1449,58 +1408,74 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workSpace */ - /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); - size_t const bufferSpace = buffInSize + buffOutSize; + ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue; + + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + needsIndexReset = ZSTDirp_reset; + } + + ZSTD_cwksp_bump_oversized_duration(ws, 0); + + /* Check if workspace is large enough, alloc a new one if needed */ + { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize); size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); - size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)); - size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace + - ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; - int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; - int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0; + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); - if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", - zc->workSpaceSize >> 10, + if (workspaceTooSmall || workspaceWasteful) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workSpaceSize = 0; - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation); - zc->workSpaceSize = neededSpace; - zc->workSpaceOversizedDuration = 0; + needsIndexReset = ZSTDirp_reset; + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); + + DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ - assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; - zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; - ptr = zc->blockState.nextCBlock + 1; - zc->entropyWorkspace = (U32*)ptr; + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, HUF_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } + ZSTD_cwksp_clear(ws); + /* init params */ zc->appliedParams = params; zc->blockState.matchState.cParams = params.cParams; @@ -1519,58 +1494,58 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); - - /* ldm hash table */ - /* initialize bucketOffsets table later for pointer alignment */ - if (params.ldmParams.enableLdm) { - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->ldmState.hashTable = (ldmEntry_t*)ptr; - ptr = zc->ldmState.hashTable + ldmHSize; - zc->ldmSequences = (rawSeq*)ptr; - ptr = zc->ldmSequences + maxNbLdmSeq; - zc->maxNbLdmSequences = maxNbLdmSeq; - - memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); - } - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE*) ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; - ptr = zc->seqStore.litStart + blockSize + WILDCOPY_OVERLENGTH; - - /* ldm bucketOffsets table */ - if (params.ldmParams.enableLdm) { - size_t const ldmBucketSize = - ((size_t)1) << (params.ldmParams.hashLog - - params.ldmParams.bucketSizeLog); - memset(ptr, 0, ldmBucketSize); - zc->ldmState.bucketOffsets = (BYTE*)ptr; - ptr = zc->ldmState.bucketOffsets + ldmBucketSize; - ZSTD_window_clear(&zc->ldmState.window); - } - ZSTD_referenceExternalSequences(zc, NULL, 0); /* buffers */ zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ptr; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); zc->outBuffSize = buffOutSize; - zc->outBuff = zc->inBuff + buffInSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ + size_t const ldmBucketSize = + ((size_t)1) << (params.ldmParams.hashLog - + params.ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms.cParams, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx)); + + /* ldm hash table */ + if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); + ZSTD_window_clear(&zc->ldmState.window); + } + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); return 0; } @@ -1604,15 +1579,15 @@ static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { }; static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize) { size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; return ( pledgedSrcSize <= cutoff || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params.attachDictPref == ZSTD_dictForceAttach ) - && params.attachDictPref != ZSTD_dictForceCopy - && !params.forceWindow; /* dictMatchState isn't correctly + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow; /* dictMatchState isn't correctly * handled in _enforceMaxDist */ } @@ -1630,8 +1605,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, * has its own tables. */ params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1679,30 +1654,36 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* Copy only compression parameters related to tables. */ params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_leaveDirty, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); } + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + /* copy tables */ { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - size_t const tableSpace = (chainSize + hSize) * sizeof(U32); - assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); - assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); - memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); } /* Zero the hashTable3, since the cdict never fills it */ - { size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3; + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + /* copy dictionary offsets */ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; @@ -1724,7 +1705,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, * in-place. We decide here which strategy to use. */ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { @@ -1734,10 +1715,10 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { return ZSTD_resetCCtx_byAttachingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } else { return ZSTD_resetCCtx_byCopyingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } } @@ -1763,7 +1744,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, params.cParams = srcCCtx->appliedParams.cParams; params.fParams = fParams; ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); @@ -1771,16 +1752,27 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); } + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + /* copy tables */ { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->blockState.matchState.hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); - memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); } + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + /* copy dictionary offsets */ { const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; @@ -1831,6 +1823,20 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa int rowNb; assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ assert(size < (1U<<31)); /* can be casted to int */ + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { int column; for (column=0; columncParams.windowLog > STREAM_ACCUMULATOR_MIN; @@ -1971,7 +1977,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, - workspace, wkspSize, + entropyWorkspace, entropyWkspSize, bmi2); FORWARD_IF_ERROR(cSize); assert(cSize <= dstCapacity); @@ -1981,12 +1987,17 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* Sequences Header */ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, dstSize_tooSmall); - if (nbSeq < 0x7F) + if (nbSeq < 128) { *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else - op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ @@ -2002,7 +2013,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building LL table"); nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, @@ -2012,10 +2023,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (LLtype == set_compressed) lastNCount = op; @@ -2024,7 +2039,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for Offsets */ { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); @@ -2035,10 +2051,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (Offtype == set_compressed) lastNCount = op; @@ -2047,7 +2067,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for MatchLengths */ { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, @@ -2056,10 +2077,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, (size_t)(oend - op), CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (MLtype == set_compressed) lastNCount = op; @@ -2107,13 +2132,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { size_t const cSize = ZSTD_compressSequences_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, - workspace, wkspSize, bmi2); + entropyWorkspace, entropyWkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. @@ -2264,11 +2289,99 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) return ZSTDbss_compress; } +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqs = seqStore->sequencesStart; + size_t seqsSize = seqStore->sequences - seqs; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; size_t position; int repIdx; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= ZSTD_REP_NUM) { + outSeqs[i].rep = outSeqs[i].offset; + repIdx = (unsigned int)i - outSeqs[i].offset; + + if (outSeqs[i].litLength == 0) { + if (outSeqs[i].offset < 3) { + --repIdx; + } else { + repIdx = (unsigned int)i - 1; + } + ++outSeqs[i].rep; + } + assert(repIdx >= -3); + outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; + if (outSeqs[i].rep == 4) { + --outSeqs[i].offset; + } + } else { + outSeqs[i].offset -= ZSTD_REP_NUM; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = (unsigned int)position; + position += outSeqs[i].matchLength; + } + zc->seqCollector.seqIndex += seqsSize; +} + +size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +/* Returns true if the given block is a RLE block */ +static int ZSTD_isRLE(const BYTE *ip, size_t length) { + size_t i; + if (length < 2) return 1; + for (i = 1; i < length; ++i) { + if (ip[0] != ip[i]) return 0; + } + return 1; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 frame) { + /* This the upper bound for the length of an rle block. + * This isn't the actual upper bound. Finding the real threshold + * needs further investigation. + */ + const U32 rleMaxLength = 25; size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); @@ -2278,6 +2391,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } } + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, @@ -2287,8 +2405,21 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + out: - if (!ZSTD_isError(cSize) && cSize != 0) { + if (!ZSTD_isError(cSize) && cSize > 1) { /* confirm repcodes and entropy tables when emitting a compressed block */ ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; zc->blockState.prevCBlock = zc->blockState.nextCBlock; @@ -2305,7 +2436,11 @@ out: } -static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) { if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { U32 const maxDist = (U32)1 << params->cParams.windowLog; @@ -2314,7 +2449,9 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; else ms->nextToUpdate -= correction; /* invalidate dictionaries on overflow correction */ @@ -2323,7 +2460,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } - /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2357,7 +2493,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ @@ -2365,15 +2502,16 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, { size_t cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, - ip, blockSize); + ip, blockSize, 1 /* frame */); FORWARD_IF_ERROR(cSize); - if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(op, cBlockHeader24); + const U32 cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } @@ -2383,6 +2521,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op += cSize; assert(dstCapacity >= cSize); dstCapacity -= cSize; + cctx->isFirstBlock = 0; DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", (unsigned)cSize); } } @@ -2393,25 +2532,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - ZSTD_CCtx_params params, U64 pledgedSrcSize, U32 dictID) + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag>0; - U32 const windowSize = (U32)1 << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); - BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = params.fParams.contentSizeFlag ? + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos=0; - assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", - !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); - if (params.format == ZSTD_f_zstd1) { + if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; } @@ -2477,7 +2616,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, "missing init (ZSTD_compressBegin)"); if (frame && (cctx->stage==ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); assert(fhSize <= dstCapacity); @@ -2497,13 +2636,15 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : - ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); FORWARD_IF_ERROR(cSize); cctx->consumedSrcSize += srcSize; cctx->producedCSize += (cSize + fhSize); @@ -2550,6 +2691,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const * @return : 0, or an error code */ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) @@ -2570,7 +2712,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); const BYTE* const ichunk = ip + chunk; - ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); switch(params->cParams.strategy) { @@ -2629,10 +2771,11 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym /*! ZSTD_loadZstdDictionary() : * @return : dictID, or an error code * assumptions : magic number supposed already checked - * dictSize supposed > 8 + * dictSize supposed >= 8 */ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, @@ -2645,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, size_t dictID; ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1< 8); + assert(dictSize >= 8); assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); dictPtr += 4; /* skip magic number */ @@ -2728,7 +2871,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; - FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, ws, params, dictPtr, dictContentSize, dtlm)); return dictID; } } @@ -2738,6 +2882,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, @@ -2745,27 +2890,35 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, void* workspace) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); - if ((dict==NULL) || (dictSize<=8)) return 0; + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); + return 0; + } ZSTD_reset_compressedBlockState(bs); /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ws, params, dict, dictSize, dtlm); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent( + ms, ws, params, dict, dictSize, dtlm); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); assert(0); /* impossible */ } /* dict as full zstd dictionary */ - return ZSTD_loadZstdDictionary(bs, ms, params, dict, dictSize, dtlm, workspace); + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, workspace); } +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6) + /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, @@ -2773,23 +2926,34 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, U64 pledgedSrcSize, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (cdict && cdict->dictContentSize>0) { + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff) ); - { size_t const dictID = ZSTD_compress_insertDictionary( - cctx->blockState.prevCBlock, &cctx->blockState.matchState, - ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff) ); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, + dictContentType, dtlm, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->workspace, params, dict, dictSize, + dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -2802,12 +2966,12 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); /* compression parameters verification and optimization */ - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, dtlm, cdict, @@ -2822,21 +2986,21 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL /*cdict*/, - cctxParams, pledgedSrcSize); + &cctxParams, pledgedSrcSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, - cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) @@ -2859,7 +3023,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); FORWARD_IF_ERROR(fhSize); dstCapacity -= fhSize; op += fhSize; @@ -2920,13 +3084,13 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, ZSTD_parameters params) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compress_internal"); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - cctxParams); + &cctxParams); } size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, @@ -2950,7 +3114,7 @@ size_t ZSTD_compress_advanced_internal( void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params) + const ZSTD_CCtx_params* params) { DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -2966,9 +3130,9 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0); - ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); assert(params.fParams.contentSizeFlag == 1); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -3003,8 +3167,11 @@ size_t ZSTD_estimateCDictSize_advanced( ZSTD_dictLoadMethod_e dictLoadMethod) { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); - return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3017,7 +3184,9 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); } static size_t ZSTD_initCDict_internal( @@ -3031,28 +3200,29 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(cParams)); cdict->matchState.cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; } else { - void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); + cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState(&cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, - ZSTDcrp_continue, ZSTD_resetTarget_CDict); - assert(end == (char*)cdict->workspace + cdict->workspaceSize); - (void)end; - } + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + &cParams, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary - * Skips loading the dictionary if it is <= 8 bytes. + * Skips loading the dictionary if it is < 8 bytes. */ { ZSTD_CCtx_params params; memset(¶ms, 0, sizeof(params)); @@ -3060,9 +3230,9 @@ static size_t ZSTD_initCDict_internal( params.fParams.contentSizeFlag = 1; params.cParams = cParams; { size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState, ¶ms, - cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->workspace); + &cdict->cBlockState, &cdict->matchState, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -3080,18 +3250,29 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; - if (!cdict || !workspace) { - ZSTD_free(cdict, customMem); + if (!workspace) { ZSTD_free(workspace, customMem); return NULL; } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; - cdict->workspace = workspace; - cdict->workspaceSize = workspaceSize; + cdict->compressionLevel = 0; /* signals advanced API usage */ + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3107,9 +3288,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) @@ -3124,9 +3308,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_free(cdict->workspace, cMem); - ZSTD_free(cdict->dictBuffer, cMem); - ZSTD_free(cdict, cMem); + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_free(cdict, cMem); + } return 0; } } @@ -3152,28 +3338,30 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) - + HUF_WORKSPACE_SIZE + matchStateSize; - ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; - void* ptr; + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(cdict+1, dict, dictSize); - dict = cdict+1; - ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; - } else { - ptr = cdict+1; - } - cdict->workspace = ptr; - cdict->workspaceSize = HUF_WORKSPACE_SIZE + matchStateSize; - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, - ZSTD_dlm_byRef, dictContentType, + dictLoadMethod, dictContentType, cParams) )) return NULL; @@ -3195,7 +3383,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) + && (params.attachDictPref != ZSTD_dictForceLoad) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. @@ -3209,7 +3405,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( return ZSTD_compressBegin_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_not_buffered); } } @@ -3300,7 +3496,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_buffered) ); cctx->inToCompress = 0; @@ -3334,13 +3530,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - zcs->requestedParams = params; + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict) { FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); @@ -3379,7 +3576,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) /* ZSTD_initCStream_advanced() : * pledgedSrcSize must be exact. * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pss) @@ -3393,7 +3590,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, params); FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); return 0; } @@ -3643,7 +3840,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, if (cctx->mtctx == NULL) { DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); - cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem); RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation); } /* mt compression */ @@ -3771,8 +3968,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ - { 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ - { 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ @@ -3796,8 +3993,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /* W, C, H, S, L, T, strat */ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */ - { 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ @@ -3823,8 +4020,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ - { 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */ - { 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ @@ -3849,7 +4046,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ - { 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index 6d623cc6be8..14036f873f5 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -19,6 +19,7 @@ * Dependencies ***************************************/ #include "zstd_internal.h" +#include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif @@ -192,6 +193,13 @@ typedef struct { size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + struct ZSTD_CCtx_params_s { ZSTD_format_e format; ZSTD_compressionParameters cParams; @@ -203,6 +211,9 @@ struct ZSTD_CCtx_params_s { size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize. * No target when targetCBlockSize == 0. * There is no guarantee on compressed block size */ + int srcSizeHint; /* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size */ ZSTD_dictAttachPref_e attachDictPref; ZSTD_literalCompressionMode_e literalCompressionMode; @@ -228,9 +239,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workSpaceOversizedDuration; - void* workSpace; - size_t workSpaceSize; + ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; @@ -238,6 +247,8 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; + SeqCollector seqCollector; + int isFirstBlock; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ @@ -337,26 +348,57 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) return (srcSize >> minlog) + 2; } +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + /*! ZSTD_storeSeq() : - * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. - * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). + * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. + * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). * `mlBase` : matchLength - MINMATCH + * Allowed to overread literals up to litLimit. */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase) +HINT_INLINE UNUSED_ATTR +void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ { U32 const pos = (U32)((const BYTE*)literals - g_start); DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode); + pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); } #endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, (ptrdiff_t)litLength, ZSTD_no_overlap); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } seqStorePtr->lit += litLength; /* literal Length */ @@ -368,7 +410,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ - seqStorePtr->sequences[0].offset = offsetCode + 1; + seqStorePtr->sequences[0].offset = offCode + 1; /* match Length */ if (mlBase>0xFFFF) { @@ -910,7 +952,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); void ZSTD_resetSeqStore(seqStore_t* ssPtr); @@ -925,7 +967,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); /* ZSTD_compress_advanced_internal() : @@ -934,7 +976,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params); + const ZSTD_CCtx_params* params); /* ZSTD_writeLastEmptyBlock() : diff --git a/thirdparty/zstd/compress/zstd_compress_literals.c b/thirdparty/zstd/compress/zstd_compress_literals.c index eb3e5a44bc0..6c133311821 100644 --- a/thirdparty/zstd/compress/zstd_compress_literals.c +++ b/thirdparty/zstd/compress/zstd_compress_literals.c @@ -70,7 +70,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2) { size_t const minGain = ZSTD_minGain(srcSize, strategy); @@ -99,10 +99,15 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, { HUF_repeat repeat = prevHuf->repeatMode; int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat_none) { /* reused the existing table */ hType = set_repeat; diff --git a/thirdparty/zstd/compress/zstd_compress_literals.h b/thirdparty/zstd/compress/zstd_compress_literals.h index 7adbecc0beb..97273d7cfdb 100644 --- a/thirdparty/zstd/compress/zstd_compress_literals.h +++ b/thirdparty/zstd/compress/zstd_compress_literals.h @@ -23,7 +23,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, const int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.c b/thirdparty/zstd/compress/zstd_compress_sequences.c index 3c3deae08cb..0ff7a26823b 100644 --- a/thirdparty/zstd/compress/zstd_compress_sequences.c +++ b/thirdparty/zstd/compress/zstd_compress_sequences.c @@ -222,7 +222,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize) + void* entropyWorkspace, size_t entropyWorkspaceSize) { BYTE* op = (BYTE*)dst; const BYTE* const oend = op + dstCapacity; @@ -238,7 +238,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, memcpy(nextCTable, prevCTable, prevCTableSize); return 0; case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */ return 0; case set_compressed: { S16 norm[MaxSeq + 1]; @@ -252,7 +252,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); return NCountSize; } } diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.h b/thirdparty/zstd/compress/zstd_compress_sequences.h index f5234d94c81..57e8e367b09 100644 --- a/thirdparty/zstd/compress/zstd_compress_sequences.h +++ b/thirdparty/zstd/compress/zstd_compress_sequences.h @@ -35,7 +35,7 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize); + void* entropyWorkspace, size_t entropyWorkspaceSize); size_t ZSTD_encodeSequences( void* dst, size_t dstCapacity, diff --git a/thirdparty/zstd/compress/zstd_cwksp.h b/thirdparty/zstd/compress/zstd_cwksp.h new file mode 100644 index 00000000000..fc9765bd3f3 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_cwksp.h @@ -0,0 +1,535 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "zstd_internal.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + + int allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +MEM_STATIC void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + ws->tableValidEnd = ws->objectEnd; + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + if (ws->allocStart < ws->tableValidEnd) { + ws->tableValidEnd = ws->allocStart; + } + } + ws->phase = phase; + } +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on sizeof(unsigned). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_internal_advance_phase(ws, phase); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)alloc & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + __msan_poison(ws->objectEnd, size); + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace (or at least the non-static, non-table parts of it) + * every time we start a new compression. */ + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } +#endif + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_free(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before its used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 54467cc31bd..a661a48534d 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -148,7 +148,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -157,7 +157,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -247,7 +247,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -278,7 +278,7 @@ _match_stored: const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -297,7 +297,7 @@ _match_stored: U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -411,7 +411,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -422,7 +422,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -447,7 +447,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -479,7 +479,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index 59267ffbbcf..6dbefee6b7f 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -8,7 +8,7 @@ * You may select, at your option, one of the above-listed licenses. */ -#include "zstd_compress_internal.h" +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ #include "zstd_fast.h" @@ -43,8 +43,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_fast_generic( +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls) @@ -74,8 +74,7 @@ size_t ZSTD_compressBlock_fast_generic( DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; - { - U32 const maxRep = (U32)(ip0 - prefixStart); + { U32 const maxRep = (U32)(ip0 - prefixStart); if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -118,8 +117,7 @@ size_t ZSTD_compressBlock_fast_generic( match0 = match1; goto _offset; } - { - size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize; + { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; assert(step >= 2); ip0 += step; ip1 += step; @@ -138,7 +136,7 @@ _offset: /* Requires: ip0, match0 */ _match: /* Requires: ip0, match0, offcode */ /* Count the forward length */ mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; - ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); /* match found */ ip0 += mLength; anchor = ip0; @@ -150,16 +148,15 @@ _match: /* Requires: ip0, match0, offcode */ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - while ( (ip0 <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) { + while ( ((ip0 <= ilimit) & (offset_2>0)) /* offset_2==0 means offset_2 is invalidated */ + && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { /* store sequence */ size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); ip0 += rLength; ip1 = ip0 + 1; - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); anchor = ip0; continue; /* faster when present (confirmed on gcc-8) ... (?) */ } @@ -179,8 +176,7 @@ size_t ZSTD_compressBlock_fast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState == NULL); switch(mls) { @@ -265,7 +261,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -285,7 +281,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -300,7 +296,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -325,7 +321,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -348,8 +344,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState != NULL); switch(mls) { @@ -408,16 +403,17 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const U32 repIndex = current + 1 - offset_1; const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - size_t mLength; hashTable[h] = current; /* update hash table */ assert(offset_1 <= current +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; + size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -427,19 +423,15 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( } { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + U32 const offset = current - matchIndex; + size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset = current - matchIndex; - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + offset_2 = offset_1; offset_1 = offset; /* update offset history */ + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ip += mLength; + anchor = ip; } } - /* found a match : store it */ - ip += mLength; - anchor = ip; - if (ip <= ilimit) { /* Fill Table */ hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; @@ -448,13 +440,13 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -476,8 +468,7 @@ size_t ZSTD_compressBlock_fast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 0af41724c7c..9ad7e03b54c 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -810,7 +810,7 @@ ZSTD_compressBlock_lazy_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -828,7 +828,7 @@ _storeSequence: const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; @@ -843,7 +843,7 @@ _storeSequence: /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -1051,7 +1051,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -1066,7 +1066,7 @@ _storeSequence: const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 3dcf86e6e8a..c3312ad3e3d 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params) { size_t const ldmHSize = ((size_t)1) << params.hashLog; size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); - size_t const ldmBucketSize = - ((size_t)1) << (params.hashLog - ldmBucketSizeLog); - size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); return params.enableLdm ? totalSize : 0; } @@ -583,7 +583,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, rep[i] = rep[i-1]; rep[0] = sequence.offset; /* Store the sequence */ - ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, sequence.offset + ZSTD_REP_MOVE, sequence.matchLength - MINMATCH); ip += sequence.matchLength; diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index 2da363f93ef..2e50fca6ff5 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -1098,7 +1098,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ assert(anchor + llen <= iend); ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, offCode, mlen-MINMATCH); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); anchor += advance; ip = anchor; } } diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 9e537b88485..bc3062b5309 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -668,7 +668,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) /* init */ if (job->cdict) { - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize); + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); assert(job->firstJob); /* only allowed for first job */ if (ZSTD_isError(initError)) JOB_ERROR(initError); } else { /* srcStart points at reloaded section */ @@ -680,7 +680,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ ZSTD_dtlm_fast, NULL, /*cdict*/ - jobParams, pledgedSrcSize); + &jobParams, pledgedSrcSize); if (ZSTD_isError(initError)) JOB_ERROR(initError); } } @@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) unsigned jobID; DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); - mtctx->jobs[jobID].dstBuff = g_nullBuffer; - mtctx->jobs[jobID].cSize = 0; + + /* Clear the job description, but keep the mutex/cond */ + memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; } - memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription)); mtctx->inBuff.buffer = g_nullBuffer; mtctx->inBuff.filled = 0; mtctx->allJobsCompleted = 1; @@ -1028,9 +1034,9 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, /* Sets parameters relevant to the compression job, * initializing others to default values. */ -static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) +static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params) { - ZSTD_CCtx_params jobParams = params; + ZSTD_CCtx_params jobParams = *params; /* Clear parameters related to multithreading */ jobParams.forceWindow = 0; jobParams.nbWorkers = 0; @@ -1151,16 +1157,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) /* ===== Multi-threaded compression ===== */ /* ------------------------------------------ */ -static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) { unsigned jobLog; - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - jobLog = MAX(21, params.cParams.chainLog + 4); + jobLog = MAX(21, params->cParams.chainLog + 4); } else { - jobLog = MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(20, params->cParams.windowLog + 2); } return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } @@ -1193,27 +1199,27 @@ static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) return ovlog; } -static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) { - int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy); - int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog); + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); assert(0 <= overlapRLog && overlapRLog <= 8); - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ - ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "overlapLog : %i", params.overlapLog); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; } static unsigned -ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) +ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nbWorkers) { assert(nbWorkers>0); { size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); @@ -1236,9 +1242,9 @@ static size_t ZSTDMT_compress_advanced_internal( const ZSTD_CDict* cdict, ZSTD_CCtx_params params) { - ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); - size_t const overlapSize = ZSTDMT_computeOverlapSize(params); - unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers); + ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(¶ms); + size_t const overlapSize = ZSTDMT_computeOverlapSize(¶ms); + unsigned const nbJobs = ZSTDMT_computeNbJobs(¶ms, srcSize, params.nbWorkers); size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs; size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */ const char* const srcStart = (const char*)src; @@ -1256,7 +1262,7 @@ static size_t ZSTDMT_compress_advanced_internal( ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode"); if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, jobParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, &jobParams); } assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ @@ -1404,12 +1410,12 @@ size_t ZSTDMT_initCStream_internal( mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { - ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); + ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(¶ms); DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); assert(singleThreadParams.nbWorkers == 0); return ZSTD_initCStream_internal(mtctx->cctxPool->cctx[0], dict, dictSize, cdict, - singleThreadParams, pledgedSrcSize); + &singleThreadParams, pledgedSrcSize); } DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); @@ -1435,11 +1441,11 @@ size_t ZSTDMT_initCStream_internal( mtctx->cdict = cdict; } - mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params); + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; if (mtctx->targetSectionSize == 0) { - mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); } assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); diff --git a/thirdparty/zstd/decompress/huf_decompress.c b/thirdparty/zstd/decompress/huf_decompress.c index 3f8bd297320..bb2d0a96bc1 100644 --- a/thirdparty/zstd/decompress/huf_decompress.c +++ b/thirdparty/zstd/decompress/huf_decompress.c @@ -61,7 +61,9 @@ * Error Management ****************************************************************/ #define HUF_isError ERR_isError +#ifndef CHECK_F #define CHECK_F(f) { size_t const err_ = (f); if (HUF_isError(err_)) return err_; } +#endif /* ************************************************************** diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index 751060b2cd1..dd4591b7be9 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -88,10 +88,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } static size_t ZSTD_startingInputLength(ZSTD_format_e format) { - size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? - ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE : - ZSTD_FRAMEHEADERSIZE_PREFIX; - ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); return startingInputLength; @@ -376,7 +373,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; - while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { U32 const magicNumber = MEM_readLE32(src); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -629,11 +626,12 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /* check */ RETURN_ERROR_IF( - remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize, + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, srcSize_wrong); /* Frame Header */ - { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, srcSize_wrong); @@ -714,7 +712,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, dictSize = ZSTD_DDict_dictSize(ddict); } - while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) { @@ -1098,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); for (i=0; i<3; i++) { U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; - RETURN_ERROR_IF(rep==0 || rep >= dictContentSize, + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, dictionary_corrupted); entropy->rep[i] = rep; } } @@ -1267,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, { RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); ZSTD_clearDict(dctx); - if (dict && dictSize >= 8) { + if (dict && dictSize != 0) { dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); dctx->ddict = dctx->ddictLocal; @@ -1300,14 +1298,14 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz /* ZSTD_initDStream_usingDict() : - * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * return : expected size, aka ZSTD_startingInputLength(). * this function cannot fail */ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) { DEBUGLOG(4, "ZSTD_initDStream_usingDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(zds->format); } /* note : this variant can't fail */ @@ -1324,16 +1322,16 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) ); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(dctx->format); } /* ZSTD_resetDStream() : - * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * return : expected size, aka ZSTD_startingInputLength(). * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(dctx->format); } @@ -1564,7 +1562,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; - return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index cbcfc084061..767e5f9a0b0 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -573,38 +573,118 @@ typedef struct { size_t pos; } seqState_t; +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +static void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} -/* ZSTD_execSequenceLast7(): - * exceptional case : decompress a match starting within last 7 bytes of output buffer. - * requires more careful checks, to ensure there is no overflow. - * performance does not matter though. - * note : this case is supposed to be never generated "naturally" by reference encoder, - * since in most cases it needs at least 8 bytes to look for a match. - * but it's allowed by the specification. */ +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op = oend_w; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer"); + /* bounds checks */ + assert(oLitEnd < oMatchEnd); + RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must fit within dstBuffer"); RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer"); /* copy literals */ - while (op < oLitEnd) *op++ = *(*litPtr)++; + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected); - match = dictEnd - (base-match); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); + match = dictEnd - (prefixStart-match); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; @@ -614,13 +694,12 @@ size_t ZSTD_execSequenceLast7(BYTE* op, memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; - match = base; + match = prefixStart; } } - while (op < oMatchEnd) *op++ = *match++; + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } - HINT_INLINE size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, @@ -634,20 +713,29 @@ size_t ZSTD_execSequence(BYTE* op, const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + /* Errors and uncommon cases handled here. */ + assert(oLitEnd < oMatchEnd); + if (iLitEnd > litLimit || oMatchEnd > oend_w) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - else - ZSTD_copy8(op, *litPtr); + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (sequence.litLength > 16) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ - /* copy Match */ + /* Copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); @@ -662,123 +750,33 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } } } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (sequence.offset >= WILDCOPY_VECLEN) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; } - op += 8; match += 8; + assert(sequence.offset < WILDCOPY_VECLEN); - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ - } - return sequenceLength; -} + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); - -HINT_INLINE -size_t ZSTD_execSequenceLong(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = sequence.match; - - /* check */ - RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); - - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - else - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ - - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } - } } - assert(op <= oend_w); - assert(sequence.matchLength >= MINMATCH); - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); } return sequenceLength; } @@ -1098,7 +1096,7 @@ ZSTD_decompressSequencesLong_body( /* decode and decompress */ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb /* INT_MAX */ #include /* size_t */ @@ -71,7 +72,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 3 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -196,9 +197,13 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx - * The function will compress at requested compression level, - * ignoring any other parameter */ + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -233,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, * using ZSTD_CCtx_set*() functions. * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . * * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * @@ -261,18 +266,26 @@ typedef enum { /* compression parameters * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict() - * for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level sets all default values of other compression parameters */ + * Note 2 : setting a level resets all other compression parameters to default */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. * Special: value 0 means "use default windowLog". * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such window size at decompression stage if using streaming. */ + * requires explicitly allowing such size at streaming decompression stage. */ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. * Resulting memory usage is (1 << (hashLog+2)). * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. @@ -283,13 +296,13 @@ typedef enum { * Resulting memory usage is (1 << (chainLog+2)). * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. * Larger tables result in better and slower compression. - * This parameter is useless when using "fast" strategy. + * This parameter is useless for "fast" strategy. * It's still useful when using "dfast" strategy, * in which case it defines a secondary probe table. * Special: value 0 means "use default chainLog". */ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. * More attempts result in better and slower compression. - * This parameter is useless when using "fast" and "dFast" strategies. + * This parameter is useless for "fast" and "dFast" strategies. * Special: value 0 means "use default searchLog". */ ZSTD_c_minMatch=105, /* Minimum size of searched matches. * Note that Zstandard can still find matches of smaller size, @@ -344,7 +357,7 @@ typedef enum { ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) * Content size must be known at the beginning of compression. * This is automatically the case when using ZSTD_compress2(), - * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ @@ -363,7 +376,7 @@ typedef enum { * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced */ + * The minimum size is automatically and transparently enforced. */ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. * It helps preserve compression ratio, while each job is compressed in parallel. @@ -386,6 +399,7 @@ typedef enum { * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -396,6 +410,7 @@ typedef enum { ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004 } ZSTD_cParameter; typedef struct { @@ -793,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, typedef struct ZSTD_CDict_s ZSTD_CDict; /*! ZSTD_createCDict() : - * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. - * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. - * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); @@ -925,7 +945,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. * It's a CPU consuming operation, with non-negligible impact on latency. * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); @@ -969,7 +989,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. * Prefix buffer must remain unmodified up to the end of frame, * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. * A full dictionary is more costly, as it requires building tables. @@ -1014,8 +1034,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * Some of them might be removed in the future (especially when redundant with existing stable functions) * ***************************************************************************************/ -#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN 6 +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ #define ZSTD_SKIPPABLEHEADERSIZE 8 @@ -1063,6 +1083,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); /* Advanced parameter bounds */ #define ZSTD_TARGETCBLOCKSIZE_MIN 64 #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_SRCSIZEHINT_MIN 0 +#define ZSTD_SRCSIZEHINT_MAX INT_MAX /* internal */ #define ZSTD_HASHLOG3_MAX 17 @@ -1072,6 +1094,24 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; +typedef struct { + unsigned int matchPos; /* Match pos in dst */ + /* If seqDef.offset > 3, then this is seqDef.offset - 3 + * If seqDef.offset < 3, then this is the corresponding repeat offset + * But if seqDef.offset < 3 and litLength == 0, this is the + * repeat offset before the corresponding repeat offset + * And if seqDef.offset == 3 and litLength == 0, this is the + * most recent repeat offset - 1 + */ + unsigned int offset; + unsigned int litLength; /* Literal length */ + unsigned int matchLength; /* Match length */ + /* 0 when seq not rep and seqDef.offset otherwise + * when litLength == 0 this will be <= 4, otherwise <= 3 like normal + */ + unsigned int rep; +} ZSTD_Sequence; + typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ @@ -1101,21 +1141,12 @@ typedef enum { typedef enum { ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e; typedef enum { - /* Opened question : should we have a format ZSTD_f_auto ? - * Today, it would mean exactly the same as ZSTD_f_zstd1. - * But, in the future, should several formats become supported, - * on the compression side, it would mean "default format". - * On the decompression side, it would mean "automatic format detection", - * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". - * Since meaning is a little different, another option could be to define different enums for compression and decompression. - * This question could be kept for later, when there are actually multiple formats to support, - * but there is also the question of pinning enum values, and pinning value `0` is especially important */ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. * Useful to save 4 bytes per generated frame. * Decoder cannot recognise automatically this format, requiring this instruction. */ } ZSTD_format_e; @@ -1126,7 +1157,7 @@ typedef enum { * to evolve and should be considered only in the context of extremely * advanced performance tuning. * - * Zstd currently supports the use of a CDict in two ways: + * Zstd currently supports the use of a CDict in three ways: * * - The contents of the CDict can be copied into the working context. This * means that the compression can search both the dictionary and input @@ -1142,6 +1173,12 @@ typedef enum { * working context's tables can be reused). For small inputs, this can be * faster than copying the CDict's tables. * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that * Zstd is making poor choices, it is possible to override that choice with @@ -1150,6 +1187,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum { @@ -1158,7 +1196,7 @@ typedef enum { * levels will be compressed. */ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; @@ -1210,20 +1248,38 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +/*! ZSTD_getSequences() : + * Extract sequences from the sequence store + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * @return : number of sequences extracted + */ +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); + /*************************************** * Memory management ***************************************/ /*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CCtx size estimation is only correct for single-threaded compression. */ + * These functions make it possible to estimate memory usage of a future + * {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a budget large enough for any + * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(), + * this estimate does not include space for a window buffer, so this estimate + * is guaranteed to be enough for single-shot compressions, but not streaming + * compressions. It will however assume the input may be arbitrarily large, + * which is the worst case. If srcSize is known to always be small, + * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with + * ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with + * ZSTD_CCtxParams_setParameter(). + * + * Note: only single-threaded compression is supported. This function will + * return an error code if ZSTD_c_nbWorkers is >= 1. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); @@ -1334,7 +1390,8 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS * Create a digested dictionary for compression * Dictionary content is just referenced, not duplicated. * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. */ + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : @@ -1361,7 +1418,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : - * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */ + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1369,7 +1428,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ + * Note : this function is now REDUNDANT. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning in some future version */ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1441,6 +1502,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * There is no guarantee on compressed block size (default:0) */ #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1613,8 +1680,13 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + /**! ZSTD_initCStream_usingDict() : * This function is deprecated, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); @@ -1623,42 +1695,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe * * Creates of an internal CDict (incompatible with static CCtx), except if * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t +ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + /**! ZSTD_initCStream_advanced() : * This function is deprecated, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * // Pseudocode: Set each zstd parameter and leave the rest as-is. + * for ((param, value) : params) { + * ZSTD_CCtx_setParameter(zcs, param, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + /**! ZSTD_initCStream_usingCDict() : * This function is deprecated, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); * * note : cdict will just be referenced, and must outlive compression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + /**! ZSTD_initCStream_usingCDict_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. + * for ((fParam, value) : fParams) { + * ZSTD_CCtx_setParameter(zcs, fParam, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : * This function is deprecated, and is equivalent to: @@ -1673,6 +1769,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); @@ -1718,8 +1815,10 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * * note: no dictionary will be used if dict == NULL or dictSize < 8 + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + /** * This function is deprecated, and is equivalent to: * @@ -1727,14 +1826,17 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic * ZSTD_DCtx_refDDict(zds, ddict); * * note : ddict is referenced, it must outlive decompression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + /** * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * * re-use decompression parameters from previous init; saves dictionary loading + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); @@ -1908,7 +2010,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); /*! Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. A few rules to respect : From 76db2b4dcb7451d5eb06b8a9065da83d9b55c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 27 Aug 2019 13:21:15 +0200 Subject: [PATCH 012/113] Android: Fix another regression with Secure.ANDROID_ID Regression from #24145, which was missed in #28146. (cherry picked from commit 071ebb1e4871431e7edf7f679afd02e594ea5af9) --- .../android/java/{README.md => THIRDPARTY.md} | 26 ++++++++----------- .../downloader/impl/DownloaderService.java | 3 ++- .../vending/licensing/LicenseChecker.java | 3 ++- 3 files changed, 15 insertions(+), 17 deletions(-) rename platform/android/java/{README.md => THIRDPARTY.md} (63%) diff --git a/platform/android/java/README.md b/platform/android/java/THIRDPARTY.md similarity index 63% rename from platform/android/java/README.md rename to platform/android/java/THIRDPARTY.md index 58d2b107068..74ef24839b8 100644 --- a/platform/android/java/README.md +++ b/platform/android/java/THIRDPARTY.md @@ -1,5 +1,7 @@ -# Third party libraries +# Third-party libraries +This file list third-party libraries used in the Android source folder, +with their provenance and, when relevant, modifications made to those files. ## Google's vending library @@ -7,12 +9,13 @@ - Version: git (eb57657, 2018) with modifications - License: Apache 2.0 -Overwrite all files under `com/google/android/vending` +Overwrite all files under `com/google/android/vending`. -### Modify some files to avoid compile error and lint warning +Modify those files to avoid compile error and lint warning: -#### com/google/android/vending/licensing/util/Base64.java -``` +- `com/google/android/vending/licensing/util/Base64.java` + +```diff @@ -338,7 +338,8 @@ public class Base64 { e += 4; } @@ -24,8 +27,9 @@ Overwrite all files under `com/google/android/vending` } ``` -#### com/google/android/vending/licensing/LicenseChecker.java -``` +- `com/google/android/vending/licensing/LicenseChecker.java` + +```diff @@ -29,8 +29,8 @@ import android.os.RemoteException; import android.provider.Settings.Secure; import android.util.Log; @@ -37,11 +41,3 @@ Overwrite all files under `com/google/android/vending` import com.google.android.vending.licensing.util.Base64; import com.google.android.vending.licensing.util.Base64DecoderException; ``` -``` -@@ -287,13 +287,15 @@ public class LicenseChecker implements ServiceConnection { - if (logResponse) { -- String android_id = Secure.getString(mContext.getContentResolver(), -- Secure.ANDROID_ID); -+ String android_id = Secure.ANDROID_ID; - Date date = new Date(); -``` diff --git a/platform/android/java/src/com/google/android/vending/expansion/downloader/impl/DownloaderService.java b/platform/android/java/src/com/google/android/vending/expansion/downloader/impl/DownloaderService.java index 25a561ccd42..219e72c7d64 100644 --- a/platform/android/java/src/com/google/android/vending/expansion/downloader/impl/DownloaderService.java +++ b/platform/android/java/src/com/google/android/vending/expansion/downloader/impl/DownloaderService.java @@ -746,7 +746,8 @@ public abstract class DownloaderService extends CustomIntentService implements I public void run() { setServiceRunning(true); mNotification.onDownloadStateChanged(IDownloaderClient.STATE_FETCHING_URL); - String deviceId = Secure.ANDROID_ID; + String deviceId = Secure.getString(mContext.getContentResolver(), + Secure.ANDROID_ID); final APKExpansionPolicy aep = new APKExpansionPolicy(mContext, new AESObfuscator(getSALT(), mContext.getPackageName(), deviceId)); diff --git a/platform/android/java/src/com/google/android/vending/licensing/LicenseChecker.java b/platform/android/java/src/com/google/android/vending/licensing/LicenseChecker.java index 38aab9f4f52..8fc8ae86a29 100644 --- a/platform/android/java/src/com/google/android/vending/licensing/LicenseChecker.java +++ b/platform/android/java/src/com/google/android/vending/licensing/LicenseChecker.java @@ -287,7 +287,8 @@ public class LicenseChecker implements ServiceConnection { } if (logResponse) { - String android_id = Secure.ANDROID_ID; + String android_id = Secure.getString(mContext.getContentResolver(), + Secure.ANDROID_ID); Date date = new Date(); Log.d(TAG, "Server Failure: " + stringError); Log.d(TAG, "Android ID: " + android_id); From b94f72869977cd1dcd98d6ffb4b8937967a42f0a Mon Sep 17 00:00:00 2001 From: volzhs Date: Sun, 27 Oct 2019 21:29:43 +0900 Subject: [PATCH 013/113] Update libwebp to 1.0.3 (cherry picked from commit 074d421dca94ec4ddadfbfa62268bda219bab46c) --- thirdparty/README.md | 2 +- thirdparty/libwebp/src/dec/quant_dec.c | 17 ++- thirdparty/libwebp/src/dec/tree_dec.c | 57 ++++---- thirdparty/libwebp/src/dec/vp8_dec.c | 81 ++++++------ thirdparty/libwebp/src/dec/vp8i_dec.h | 2 +- thirdparty/libwebp/src/dec/vp8l_dec.c | 125 +++++++++--------- thirdparty/libwebp/src/demux/demux.c | 2 +- .../libwebp/src/dsp/alpha_processing_sse2.c | 4 +- thirdparty/libwebp/src/dsp/cpu.c | 4 +- thirdparty/libwebp/src/dsp/dec_sse2.c | 14 +- thirdparty/libwebp/src/dsp/enc_sse2.c | 2 +- thirdparty/libwebp/src/dsp/filters.c | 12 +- thirdparty/libwebp/src/dsp/filters_sse2.c | 16 ++- thirdparty/libwebp/src/dsp/lossless.c | 4 +- thirdparty/libwebp/src/dsp/lossless_enc.c | 21 +-- .../libwebp/src/dsp/lossless_enc_sse2.c | 4 +- .../libwebp/src/dsp/lossless_enc_sse41.c | 6 +- thirdparty/libwebp/src/dsp/quant.h | 15 +++ thirdparty/libwebp/src/dsp/rescaler.c | 16 +-- .../libwebp/src/dsp/rescaler_mips_dsp_r2.c | 16 +-- thirdparty/libwebp/src/dsp/rescaler_msa.c | 16 +-- thirdparty/libwebp/src/dsp/rescaler_neon.c | 32 ++--- thirdparty/libwebp/src/dsp/rescaler_sse2.c | 60 ++------- .../libwebp/src/enc/backward_references_enc.c | 11 +- thirdparty/libwebp/src/enc/histogram_enc.c | 5 +- thirdparty/libwebp/src/enc/predictor_enc.c | 14 +- thirdparty/libwebp/src/enc/quant_enc.c | 26 +++- thirdparty/libwebp/src/enc/vp8i_enc.h | 2 +- thirdparty/libwebp/src/mux/muxi.h | 2 +- .../libwebp/src/utils/bit_reader_inl_utils.h | 11 +- .../libwebp/src/utils/bit_reader_utils.c | 86 +++++++++++- .../libwebp/src/utils/bit_reader_utils.h | 33 ++++- .../libwebp/src/utils/bit_writer_utils.c | 2 +- .../libwebp/src/utils/color_cache_utils.h | 8 +- thirdparty/libwebp/src/utils/huffman_utils.c | 26 +++- thirdparty/libwebp/src/utils/huffman_utils.h | 2 + thirdparty/libwebp/src/utils/rescaler_utils.c | 8 +- thirdparty/libwebp/src/utils/thread_utils.c | 12 +- thirdparty/libwebp/src/utils/utils.h | 6 +- thirdparty/libwebp/src/webp/encode.h | 4 + 40 files changed, 454 insertions(+), 332 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index ff6156d42e7..561b6a2b436 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -242,7 +242,7 @@ from the Android NDK r18. ## libwebp - Upstream: https://chromium.googlesource.com/webm/libwebp/ -- Version: 1.0.2 +- Version: 1.0.3 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/libwebp/src/dec/quant_dec.c b/thirdparty/libwebp/src/dec/quant_dec.c index f07212ad736..a0ac018b0f5 100644 --- a/thirdparty/libwebp/src/dec/quant_dec.c +++ b/thirdparty/libwebp/src/dec/quant_dec.c @@ -61,12 +61,17 @@ static const uint16_t kAcTable[128] = { void VP8ParseQuant(VP8Decoder* const dec) { VP8BitReader* const br = &dec->br_; - const int base_q0 = VP8GetValue(br, 7); - const int dqy1_dc = VP8Get(br) ? VP8GetSignedValue(br, 4) : 0; - const int dqy2_dc = VP8Get(br) ? VP8GetSignedValue(br, 4) : 0; - const int dqy2_ac = VP8Get(br) ? VP8GetSignedValue(br, 4) : 0; - const int dquv_dc = VP8Get(br) ? VP8GetSignedValue(br, 4) : 0; - const int dquv_ac = VP8Get(br) ? VP8GetSignedValue(br, 4) : 0; + const int base_q0 = VP8GetValue(br, 7, "global-header"); + const int dqy1_dc = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 4, "global-header") : 0; + const int dqy2_dc = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 4, "global-header") : 0; + const int dqy2_ac = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 4, "global-header") : 0; + const int dquv_dc = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 4, "global-header") : 0; + const int dquv_ac = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 4, "global-header") : 0; const VP8SegmentHeader* const hdr = &dec->segment_hdr_; int i; diff --git a/thirdparty/libwebp/src/dec/tree_dec.c b/thirdparty/libwebp/src/dec/tree_dec.c index 3f5a957d32f..1c6fdea27cc 100644 --- a/thirdparty/libwebp/src/dec/tree_dec.c +++ b/thirdparty/libwebp/src/dec/tree_dec.c @@ -296,20 +296,21 @@ static void ParseIntraMode(VP8BitReader* const br, // to decode more than 1 keyframe. if (dec->segment_hdr_.update_map_) { // Hardcoded tree parsing - block->segment_ = !VP8GetBit(br, dec->proba_.segments_[0]) - ? VP8GetBit(br, dec->proba_.segments_[1]) - : 2 + VP8GetBit(br, dec->proba_.segments_[2]); + block->segment_ = !VP8GetBit(br, dec->proba_.segments_[0], "segments") + ? VP8GetBit(br, dec->proba_.segments_[1], "segments") + : VP8GetBit(br, dec->proba_.segments_[2], "segments") + 2; } else { block->segment_ = 0; // default for intra } - if (dec->use_skip_proba_) block->skip_ = VP8GetBit(br, dec->skip_p_); + if (dec->use_skip_proba_) block->skip_ = VP8GetBit(br, dec->skip_p_, "skip"); - block->is_i4x4_ = !VP8GetBit(br, 145); // decide for B_PRED first + block->is_i4x4_ = !VP8GetBit(br, 145, "block-size"); if (!block->is_i4x4_) { // Hardcoded 16x16 intra-mode decision tree. const int ymode = - VP8GetBit(br, 156) ? (VP8GetBit(br, 128) ? TM_PRED : H_PRED) - : (VP8GetBit(br, 163) ? V_PRED : DC_PRED); + VP8GetBit(br, 156, "pred-modes") ? + (VP8GetBit(br, 128, "pred-modes") ? TM_PRED : H_PRED) : + (VP8GetBit(br, 163, "pred-modes") ? V_PRED : DC_PRED); block->imodes_[0] = ymode; memset(top, ymode, 4 * sizeof(*top)); memset(left, ymode, 4 * sizeof(*left)); @@ -323,22 +324,25 @@ static void ParseIntraMode(VP8BitReader* const br, const uint8_t* const prob = kBModesProba[top[x]][ymode]; #if (USE_GENERIC_TREE == 1) // Generic tree-parsing - int i = kYModesIntra4[VP8GetBit(br, prob[0])]; + int i = kYModesIntra4[VP8GetBit(br, prob[0], "pred-modes")]; while (i > 0) { - i = kYModesIntra4[2 * i + VP8GetBit(br, prob[i])]; + i = kYModesIntra4[2 * i + VP8GetBit(br, prob[i], "pred-modes")]; } ymode = -i; #else // Hardcoded tree parsing - ymode = !VP8GetBit(br, prob[0]) ? B_DC_PRED : - !VP8GetBit(br, prob[1]) ? B_TM_PRED : - !VP8GetBit(br, prob[2]) ? B_VE_PRED : - !VP8GetBit(br, prob[3]) ? - (!VP8GetBit(br, prob[4]) ? B_HE_PRED : - (!VP8GetBit(br, prob[5]) ? B_RD_PRED : B_VR_PRED)) : - (!VP8GetBit(br, prob[6]) ? B_LD_PRED : - (!VP8GetBit(br, prob[7]) ? B_VL_PRED : - (!VP8GetBit(br, prob[8]) ? B_HD_PRED : B_HU_PRED))); + ymode = !VP8GetBit(br, prob[0], "pred-modes") ? B_DC_PRED : + !VP8GetBit(br, prob[1], "pred-modes") ? B_TM_PRED : + !VP8GetBit(br, prob[2], "pred-modes") ? B_VE_PRED : + !VP8GetBit(br, prob[3], "pred-modes") ? + (!VP8GetBit(br, prob[4], "pred-modes") ? B_HE_PRED : + (!VP8GetBit(br, prob[5], "pred-modes") ? B_RD_PRED + : B_VR_PRED)) : + (!VP8GetBit(br, prob[6], "pred-modes") ? B_LD_PRED : + (!VP8GetBit(br, prob[7], "pred-modes") ? B_VL_PRED : + (!VP8GetBit(br, prob[8], "pred-modes") ? B_HD_PRED + : B_HU_PRED)) + ); #endif // USE_GENERIC_TREE top[x] = ymode; } @@ -348,9 +352,9 @@ static void ParseIntraMode(VP8BitReader* const br, } } // Hardcoded UVMode decision tree - block->uvmode_ = !VP8GetBit(br, 142) ? DC_PRED - : !VP8GetBit(br, 114) ? V_PRED - : VP8GetBit(br, 183) ? TM_PRED : H_PRED; + block->uvmode_ = !VP8GetBit(br, 142, "pred-modes-uv") ? DC_PRED + : !VP8GetBit(br, 114, "pred-modes-uv") ? V_PRED + : VP8GetBit(br, 183, "pred-modes-uv") ? TM_PRED : H_PRED; } int VP8ParseIntraModeRow(VP8BitReader* const br, VP8Decoder* const dec) { @@ -514,8 +518,10 @@ void VP8ParseProba(VP8BitReader* const br, VP8Decoder* const dec) { for (b = 0; b < NUM_BANDS; ++b) { for (c = 0; c < NUM_CTX; ++c) { for (p = 0; p < NUM_PROBAS; ++p) { - const int v = VP8GetBit(br, CoeffsUpdateProba[t][b][c][p]) ? - VP8GetValue(br, 8) : CoeffsProba0[t][b][c][p]; + const int v = + VP8GetBit(br, CoeffsUpdateProba[t][b][c][p], "global-header") ? + VP8GetValue(br, 8, "global-header") : + CoeffsProba0[t][b][c][p]; proba->bands_[t][b].probas_[c][p] = v; } } @@ -524,9 +530,8 @@ void VP8ParseProba(VP8BitReader* const br, VP8Decoder* const dec) { proba->bands_ptr_[t][b] = &proba->bands_[t][kBands[b]]; } } - dec->use_skip_proba_ = VP8Get(br); + dec->use_skip_proba_ = VP8Get(br, "global-header"); if (dec->use_skip_proba_) { - dec->skip_p_ = VP8GetValue(br, 8); + dec->skip_p_ = VP8GetValue(br, 8, "global-header"); } } - diff --git a/thirdparty/libwebp/src/dec/vp8_dec.c b/thirdparty/libwebp/src/dec/vp8_dec.c index c904b529f6f..57efb690416 100644 --- a/thirdparty/libwebp/src/dec/vp8_dec.c +++ b/thirdparty/libwebp/src/dec/vp8_dec.c @@ -161,23 +161,26 @@ static int ParseSegmentHeader(VP8BitReader* br, VP8SegmentHeader* hdr, VP8Proba* proba) { assert(br != NULL); assert(hdr != NULL); - hdr->use_segment_ = VP8Get(br); + hdr->use_segment_ = VP8Get(br, "global-header"); if (hdr->use_segment_) { - hdr->update_map_ = VP8Get(br); - if (VP8Get(br)) { // update data + hdr->update_map_ = VP8Get(br, "global-header"); + if (VP8Get(br, "global-header")) { // update data int s; - hdr->absolute_delta_ = VP8Get(br); + hdr->absolute_delta_ = VP8Get(br, "global-header"); for (s = 0; s < NUM_MB_SEGMENTS; ++s) { - hdr->quantizer_[s] = VP8Get(br) ? VP8GetSignedValue(br, 7) : 0; + hdr->quantizer_[s] = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 7, "global-header") : 0; } for (s = 0; s < NUM_MB_SEGMENTS; ++s) { - hdr->filter_strength_[s] = VP8Get(br) ? VP8GetSignedValue(br, 6) : 0; + hdr->filter_strength_[s] = VP8Get(br, "global-header") ? + VP8GetSignedValue(br, 6, "global-header") : 0; } } if (hdr->update_map_) { int s; for (s = 0; s < MB_FEATURE_TREE_PROBS; ++s) { - proba->segments_[s] = VP8Get(br) ? VP8GetValue(br, 8) : 255u; + proba->segments_[s] = VP8Get(br, "global-header") ? + VP8GetValue(br, 8, "global-header") : 255u; } } } else { @@ -205,7 +208,7 @@ static VP8StatusCode ParsePartitions(VP8Decoder* const dec, size_t last_part; size_t p; - dec->num_parts_minus_one_ = (1 << VP8GetValue(br, 2)) - 1; + dec->num_parts_minus_one_ = (1 << VP8GetValue(br, 2, "global-header")) - 1; last_part = dec->num_parts_minus_one_; if (size < 3 * last_part) { // we can't even read the sizes with sz[]! That's a failure. @@ -229,21 +232,21 @@ static VP8StatusCode ParsePartitions(VP8Decoder* const dec, // Paragraph 9.4 static int ParseFilterHeader(VP8BitReader* br, VP8Decoder* const dec) { VP8FilterHeader* const hdr = &dec->filter_hdr_; - hdr->simple_ = VP8Get(br); - hdr->level_ = VP8GetValue(br, 6); - hdr->sharpness_ = VP8GetValue(br, 3); - hdr->use_lf_delta_ = VP8Get(br); + hdr->simple_ = VP8Get(br, "global-header"); + hdr->level_ = VP8GetValue(br, 6, "global-header"); + hdr->sharpness_ = VP8GetValue(br, 3, "global-header"); + hdr->use_lf_delta_ = VP8Get(br, "global-header"); if (hdr->use_lf_delta_) { - if (VP8Get(br)) { // update lf-delta? + if (VP8Get(br, "global-header")) { // update lf-delta? int i; for (i = 0; i < NUM_REF_LF_DELTAS; ++i) { - if (VP8Get(br)) { - hdr->ref_lf_delta_[i] = VP8GetSignedValue(br, 6); + if (VP8Get(br, "global-header")) { + hdr->ref_lf_delta_[i] = VP8GetSignedValue(br, 6, "global-header"); } } for (i = 0; i < NUM_MODE_LF_DELTAS; ++i) { - if (VP8Get(br)) { - hdr->mode_lf_delta_[i] = VP8GetSignedValue(br, 6); + if (VP8Get(br, "global-header")) { + hdr->mode_lf_delta_[i] = VP8GetSignedValue(br, 6, "global-header"); } } } @@ -352,8 +355,8 @@ int VP8GetHeaders(VP8Decoder* const dec, VP8Io* const io) { buf_size -= frm_hdr->partition_length_; if (frm_hdr->key_frame_) { - pic_hdr->colorspace_ = VP8Get(br); - pic_hdr->clamp_type_ = VP8Get(br); + pic_hdr->colorspace_ = VP8Get(br, "global-header"); + pic_hdr->clamp_type_ = VP8Get(br, "global-header"); } if (!ParseSegmentHeader(br, &dec->segment_hdr_, &dec->proba_)) { return VP8SetError(dec, VP8_STATUS_BITSTREAM_ERROR, @@ -378,7 +381,7 @@ int VP8GetHeaders(VP8Decoder* const dec, VP8Io* const io) { "Not a key frame."); } - VP8Get(br); // ignore the value of update_proba_ + VP8Get(br, "global-header"); // ignore the value of update_proba_ VP8ParseProba(br, dec); @@ -403,28 +406,28 @@ static const uint8_t kZigzag[16] = { // See section 13-2: http://tools.ietf.org/html/rfc6386#section-13.2 static int GetLargeValue(VP8BitReader* const br, const uint8_t* const p) { int v; - if (!VP8GetBit(br, p[3])) { - if (!VP8GetBit(br, p[4])) { + if (!VP8GetBit(br, p[3], "coeffs")) { + if (!VP8GetBit(br, p[4], "coeffs")) { v = 2; } else { - v = 3 + VP8GetBit(br, p[5]); + v = 3 + VP8GetBit(br, p[5], "coeffs"); } } else { - if (!VP8GetBit(br, p[6])) { - if (!VP8GetBit(br, p[7])) { - v = 5 + VP8GetBit(br, 159); + if (!VP8GetBit(br, p[6], "coeffs")) { + if (!VP8GetBit(br, p[7], "coeffs")) { + v = 5 + VP8GetBit(br, 159, "coeffs"); } else { - v = 7 + 2 * VP8GetBit(br, 165); - v += VP8GetBit(br, 145); + v = 7 + 2 * VP8GetBit(br, 165, "coeffs"); + v += VP8GetBit(br, 145, "coeffs"); } } else { const uint8_t* tab; - const int bit1 = VP8GetBit(br, p[8]); - const int bit0 = VP8GetBit(br, p[9 + bit1]); + const int bit1 = VP8GetBit(br, p[8], "coeffs"); + const int bit0 = VP8GetBit(br, p[9 + bit1], "coeffs"); const int cat = 2 * bit1 + bit0; v = 0; for (tab = kCat3456[cat]; *tab; ++tab) { - v += v + VP8GetBit(br, *tab); + v += v + VP8GetBit(br, *tab, "coeffs"); } v += 3 + (8 << cat); } @@ -438,24 +441,24 @@ static int GetCoeffsFast(VP8BitReader* const br, int ctx, const quant_t dq, int n, int16_t* out) { const uint8_t* p = prob[n]->probas_[ctx]; for (; n < 16; ++n) { - if (!VP8GetBit(br, p[0])) { + if (!VP8GetBit(br, p[0], "coeffs")) { return n; // previous coeff was last non-zero coeff } - while (!VP8GetBit(br, p[1])) { // sequence of zero coeffs + while (!VP8GetBit(br, p[1], "coeffs")) { // sequence of zero coeffs p = prob[++n]->probas_[0]; if (n == 16) return 16; } { // non zero coeff const VP8ProbaArray* const p_ctx = &prob[n + 1]->probas_[0]; int v; - if (!VP8GetBit(br, p[2])) { + if (!VP8GetBit(br, p[2], "coeffs")) { v = 1; p = p_ctx[1]; } else { v = GetLargeValue(br, p); p = p_ctx[2]; } - out[kZigzag[n]] = VP8GetSigned(br, v) * dq[n > 0]; + out[kZigzag[n]] = VP8GetSigned(br, v, "coeffs") * dq[n > 0]; } } return 16; @@ -468,24 +471,24 @@ static int GetCoeffsAlt(VP8BitReader* const br, int ctx, const quant_t dq, int n, int16_t* out) { const uint8_t* p = prob[n]->probas_[ctx]; for (; n < 16; ++n) { - if (!VP8GetBitAlt(br, p[0])) { + if (!VP8GetBitAlt(br, p[0], "coeffs")) { return n; // previous coeff was last non-zero coeff } - while (!VP8GetBitAlt(br, p[1])) { // sequence of zero coeffs + while (!VP8GetBitAlt(br, p[1], "coeffs")) { // sequence of zero coeffs p = prob[++n]->probas_[0]; if (n == 16) return 16; } { // non zero coeff const VP8ProbaArray* const p_ctx = &prob[n + 1]->probas_[0]; int v; - if (!VP8GetBitAlt(br, p[2])) { + if (!VP8GetBitAlt(br, p[2], "coeffs")) { v = 1; p = p_ctx[1]; } else { v = GetLargeValue(br, p); p = p_ctx[2]; } - out[kZigzag[n]] = VP8GetSigned(br, v) * dq[n > 0]; + out[kZigzag[n]] = VP8GetSigned(br, v, "coeffs") * dq[n > 0]; } } return 16; diff --git a/thirdparty/libwebp/src/dec/vp8i_dec.h b/thirdparty/libwebp/src/dec/vp8i_dec.h index 2d7900aae12..3de8d86f903 100644 --- a/thirdparty/libwebp/src/dec/vp8i_dec.h +++ b/thirdparty/libwebp/src/dec/vp8i_dec.h @@ -32,7 +32,7 @@ extern "C" { // version numbers #define DEC_MAJ_VERSION 1 #define DEC_MIN_VERSION 0 -#define DEC_REV_VERSION 2 +#define DEC_REV_VERSION 3 // YUV-cache parameters. Cache is 32-bytes wide (= one cacheline). // Constraints are: We need to store one 16x16 block of luma samples (y), diff --git a/thirdparty/libwebp/src/dec/vp8l_dec.c b/thirdparty/libwebp/src/dec/vp8l_dec.c index 333bb3e80d7..d3e27119ea4 100644 --- a/thirdparty/libwebp/src/dec/vp8l_dec.c +++ b/thirdparty/libwebp/src/dec/vp8l_dec.c @@ -362,12 +362,8 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, VP8LMetadata* const hdr = &dec->hdr_; uint32_t* huffman_image = NULL; HTreeGroup* htree_groups = NULL; - // When reading htrees, some might be unused, as the format allows it. - // We will still read them but put them in this htree_group_bogus. - HTreeGroup htree_group_bogus; HuffmanCode* huffman_tables = NULL; - HuffmanCode* huffman_tables_bogus = NULL; - HuffmanCode* next = NULL; + HuffmanCode* huffman_table = NULL; int num_htree_groups = 1; int num_htree_groups_max = 1; int max_alphabet_size = 0; @@ -418,12 +414,6 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, if (*mapped_group == -1) *mapped_group = num_htree_groups++; huffman_image[i] = *mapped_group; } - huffman_tables_bogus = (HuffmanCode*)WebPSafeMalloc( - table_size, sizeof(*huffman_tables_bogus)); - if (huffman_tables_bogus == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - goto Error; - } } else { num_htree_groups = num_htree_groups_max; } @@ -453,63 +443,71 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, goto Error; } - next = huffman_tables; + huffman_table = huffman_tables; for (i = 0; i < num_htree_groups_max; ++i) { - // If the index "i" is unused in the Huffman image, read the coefficients - // but store them to a bogus htree_group. - const int is_bogus = (mapping != NULL && mapping[i] == -1); - HTreeGroup* const htree_group = - is_bogus ? &htree_group_bogus : - &htree_groups[(mapping == NULL) ? i : mapping[i]]; - HuffmanCode** const htrees = htree_group->htrees; - HuffmanCode* huffman_tables_i = is_bogus ? huffman_tables_bogus : next; - int size; - int total_size = 0; - int is_trivial_literal = 1; - int max_bits = 0; - for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { - int alphabet_size = kAlphabetSize[j]; - htrees[j] = huffman_tables_i; - if (j == 0 && color_cache_bits > 0) { - alphabet_size += 1 << color_cache_bits; - } - size = - ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_tables_i); - if (size == 0) { - goto Error; - } - if (is_trivial_literal && kLiteralMap[j] == 1) { - is_trivial_literal = (huffman_tables_i->bits == 0); - } - total_size += huffman_tables_i->bits; - huffman_tables_i += size; - if (j <= ALPHA) { - int local_max_bits = code_lengths[0]; - int k; - for (k = 1; k < alphabet_size; ++k) { - if (code_lengths[k] > local_max_bits) { - local_max_bits = code_lengths[k]; - } + // If the index "i" is unused in the Huffman image, just make sure the + // coefficients are valid but do not store them. + if (mapping != NULL && mapping[i] == -1) { + for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { + int alphabet_size = kAlphabetSize[j]; + if (j == 0 && color_cache_bits > 0) { + alphabet_size += (1 << color_cache_bits); + } + // Passing in NULL so that nothing gets filled. + if (!ReadHuffmanCode(alphabet_size, dec, code_lengths, NULL)) { + goto Error; } - max_bits += local_max_bits; } - } - if (!is_bogus) next = huffman_tables_i; - htree_group->is_trivial_literal = is_trivial_literal; - htree_group->is_trivial_code = 0; - if (is_trivial_literal) { - const int red = htrees[RED][0].value; - const int blue = htrees[BLUE][0].value; - const int alpha = htrees[ALPHA][0].value; - htree_group->literal_arb = ((uint32_t)alpha << 24) | (red << 16) | blue; - if (total_size == 0 && htrees[GREEN][0].value < NUM_LITERAL_CODES) { - htree_group->is_trivial_code = 1; - htree_group->literal_arb |= htrees[GREEN][0].value << 8; + } else { + HTreeGroup* const htree_group = + &htree_groups[(mapping == NULL) ? i : mapping[i]]; + HuffmanCode** const htrees = htree_group->htrees; + int size; + int total_size = 0; + int is_trivial_literal = 1; + int max_bits = 0; + for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { + int alphabet_size = kAlphabetSize[j]; + htrees[j] = huffman_table; + if (j == 0 && color_cache_bits > 0) { + alphabet_size += (1 << color_cache_bits); + } + size = ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_table); + if (size == 0) { + goto Error; + } + if (is_trivial_literal && kLiteralMap[j] == 1) { + is_trivial_literal = (huffman_table->bits == 0); + } + total_size += huffman_table->bits; + huffman_table += size; + if (j <= ALPHA) { + int local_max_bits = code_lengths[0]; + int k; + for (k = 1; k < alphabet_size; ++k) { + if (code_lengths[k] > local_max_bits) { + local_max_bits = code_lengths[k]; + } + } + max_bits += local_max_bits; + } } + htree_group->is_trivial_literal = is_trivial_literal; + htree_group->is_trivial_code = 0; + if (is_trivial_literal) { + const int red = htrees[RED][0].value; + const int blue = htrees[BLUE][0].value; + const int alpha = htrees[ALPHA][0].value; + htree_group->literal_arb = ((uint32_t)alpha << 24) | (red << 16) | blue; + if (total_size == 0 && htrees[GREEN][0].value < NUM_LITERAL_CODES) { + htree_group->is_trivial_code = 1; + htree_group->literal_arb |= htrees[GREEN][0].value << 8; + } + } + htree_group->use_packed_table = + !htree_group->is_trivial_code && (max_bits < HUFFMAN_PACKED_BITS); + if (htree_group->use_packed_table) BuildPackedTable(htree_group); } - htree_group->use_packed_table = - !htree_group->is_trivial_code && (max_bits < HUFFMAN_PACKED_BITS); - if (htree_group->use_packed_table) BuildPackedTable(htree_group); } ok = 1; @@ -521,7 +519,6 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, Error: WebPSafeFree(code_lengths); - WebPSafeFree(huffman_tables_bogus); WebPSafeFree(mapping); if (!ok) { WebPSafeFree(huffman_image); diff --git a/thirdparty/libwebp/src/demux/demux.c b/thirdparty/libwebp/src/demux/demux.c index d8f7a40a561..ab6433e54b8 100644 --- a/thirdparty/libwebp/src/demux/demux.c +++ b/thirdparty/libwebp/src/demux/demux.c @@ -25,7 +25,7 @@ #define DMUX_MAJ_VERSION 1 #define DMUX_MIN_VERSION 0 -#define DMUX_REV_VERSION 2 +#define DMUX_REV_VERSION 3 typedef struct { size_t start_; // start location of the data diff --git a/thirdparty/libwebp/src/dsp/alpha_processing_sse2.c b/thirdparty/libwebp/src/dsp/alpha_processing_sse2.c index 76587006a12..2871c56d847 100644 --- a/thirdparty/libwebp/src/dsp/alpha_processing_sse2.c +++ b/thirdparty/libwebp/src/dsp/alpha_processing_sse2.c @@ -214,7 +214,7 @@ static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first, // Alpha detection static int HasAlpha8b_SSE2(const uint8_t* src, int length) { - const __m128i all_0xff = _mm_set1_epi8(0xff); + const __m128i all_0xff = _mm_set1_epi8((char)0xff); int i = 0; for (; i + 16 <= length; i += 16) { const __m128i v = _mm_loadu_si128((const __m128i*)(src + i)); @@ -228,7 +228,7 @@ static int HasAlpha8b_SSE2(const uint8_t* src, int length) { static int HasAlpha32b_SSE2(const uint8_t* src, int length) { const __m128i alpha_mask = _mm_set1_epi32(0xff); - const __m128i all_0xff = _mm_set1_epi8(0xff); + const __m128i all_0xff = _mm_set1_epi8((char)0xff); int i = 0; // We don't know if we can access the last 3 bytes after the last alpha // value 'src[4 * length - 4]' (because we don't know if alpha is the first diff --git a/thirdparty/libwebp/src/dsp/cpu.c b/thirdparty/libwebp/src/dsp/cpu.c index 8b40feed292..0fa5b6a5ce9 100644 --- a/thirdparty/libwebp/src/dsp/cpu.c +++ b/thirdparty/libwebp/src/dsp/cpu.c @@ -173,8 +173,8 @@ static int AndroidCPUInfo(CPUFeature feature) { const AndroidCpuFamily cpu_family = android_getCpuFamily(); const uint64_t cpu_features = android_getCpuFeatures(); if (feature == kNEON) { - return (cpu_family == ANDROID_CPU_FAMILY_ARM && - 0 != (cpu_features & ANDROID_CPU_ARM_FEATURE_NEON)); + return cpu_family == ANDROID_CPU_FAMILY_ARM && + (cpu_features & ANDROID_CPU_ARM_FEATURE_NEON) != 0; } return 0; } diff --git a/thirdparty/libwebp/src/dsp/dec_sse2.c b/thirdparty/libwebp/src/dsp/dec_sse2.c index b3840faf3a3..873aa59e8a9 100644 --- a/thirdparty/libwebp/src/dsp/dec_sse2.c +++ b/thirdparty/libwebp/src/dsp/dec_sse2.c @@ -326,7 +326,7 @@ static WEBP_INLINE void Update2Pixels_SSE2(__m128i* const pi, __m128i* const qi, const __m128i a1_lo = _mm_srai_epi16(*a0_lo, 7); const __m128i a1_hi = _mm_srai_epi16(*a0_hi, 7); const __m128i delta = _mm_packs_epi16(a1_lo, a1_hi); - const __m128i sign_bit = _mm_set1_epi8(0x80); + const __m128i sign_bit = _mm_set1_epi8((char)0x80); *pi = _mm_adds_epi8(*pi, delta); *qi = _mm_subs_epi8(*qi, delta); FLIP_SIGN_BIT2(*pi, *qi); @@ -338,9 +338,9 @@ static WEBP_INLINE void NeedsFilter_SSE2(const __m128i* const p1, const __m128i* const q0, const __m128i* const q1, int thresh, __m128i* const mask) { - const __m128i m_thresh = _mm_set1_epi8(thresh); + const __m128i m_thresh = _mm_set1_epi8((char)thresh); const __m128i t1 = MM_ABS(*p1, *q1); // abs(p1 - q1) - const __m128i kFE = _mm_set1_epi8(0xFE); + const __m128i kFE = _mm_set1_epi8((char)0xFE); const __m128i t2 = _mm_and_si128(t1, kFE); // set lsb of each byte to zero const __m128i t3 = _mm_srli_epi16(t2, 1); // abs(p1 - q1) / 2 @@ -360,7 +360,7 @@ static WEBP_INLINE void DoFilter2_SSE2(__m128i* const p1, __m128i* const p0, __m128i* const q0, __m128i* const q1, int thresh) { __m128i a, mask; - const __m128i sign_bit = _mm_set1_epi8(0x80); + const __m128i sign_bit = _mm_set1_epi8((char)0x80); // convert p1/q1 to int8_t (for GetBaseDelta_SSE2) const __m128i p1s = _mm_xor_si128(*p1, sign_bit); const __m128i q1s = _mm_xor_si128(*q1, sign_bit); @@ -380,7 +380,7 @@ static WEBP_INLINE void DoFilter4_SSE2(__m128i* const p1, __m128i* const p0, const __m128i* const mask, int hev_thresh) { const __m128i zero = _mm_setzero_si128(); - const __m128i sign_bit = _mm_set1_epi8(0x80); + const __m128i sign_bit = _mm_set1_epi8((char)0x80); const __m128i k64 = _mm_set1_epi8(64); const __m128i k3 = _mm_set1_epi8(3); const __m128i k4 = _mm_set1_epi8(4); @@ -427,7 +427,7 @@ static WEBP_INLINE void DoFilter6_SSE2(__m128i* const p2, __m128i* const p1, const __m128i* const mask, int hev_thresh) { const __m128i zero = _mm_setzero_si128(); - const __m128i sign_bit = _mm_set1_epi8(0x80); + const __m128i sign_bit = _mm_set1_epi8((char)0x80); __m128i a, not_hev; // compute hev mask @@ -941,7 +941,7 @@ static void VR4_SSE2(uint8_t* dst) { // Vertical-Right const __m128i ABCD0 = _mm_srli_si128(XABCD, 1); const __m128i abcd = _mm_avg_epu8(XABCD, ABCD0); const __m128i _XABCD = _mm_slli_si128(XABCD, 1); - const __m128i IXABCD = _mm_insert_epi16(_XABCD, I | (X << 8), 0); + const __m128i IXABCD = _mm_insert_epi16(_XABCD, (short)(I | (X << 8)), 0); const __m128i avg1 = _mm_avg_epu8(IXABCD, ABCD0); const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); diff --git a/thirdparty/libwebp/src/dsp/enc_sse2.c b/thirdparty/libwebp/src/dsp/enc_sse2.c index 7b3f142c31b..b2e78ed9411 100644 --- a/thirdparty/libwebp/src/dsp/enc_sse2.c +++ b/thirdparty/libwebp/src/dsp/enc_sse2.c @@ -777,7 +777,7 @@ static WEBP_INLINE void VR4_SSE2(uint8_t* dst, const __m128i ABCD0 = _mm_srli_si128(XABCD, 1); const __m128i abcd = _mm_avg_epu8(XABCD, ABCD0); const __m128i _XABCD = _mm_slli_si128(XABCD, 1); - const __m128i IXABCD = _mm_insert_epi16(_XABCD, I | (X << 8), 0); + const __m128i IXABCD = _mm_insert_epi16(_XABCD, (short)(I | (X << 8)), 0); const __m128i avg1 = _mm_avg_epu8(IXABCD, ABCD0); const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); diff --git a/thirdparty/libwebp/src/dsp/filters.c b/thirdparty/libwebp/src/dsp/filters.c index 069a22eaefe..9e910d99c92 100644 --- a/thirdparty/libwebp/src/dsp/filters.c +++ b/thirdparty/libwebp/src/dsp/filters.c @@ -33,9 +33,9 @@ static WEBP_INLINE void PredictLine_C(const uint8_t* src, const uint8_t* pred, uint8_t* dst, int length, int inverse) { int i; if (inverse) { - for (i = 0; i < length; ++i) dst[i] = src[i] + pred[i]; + for (i = 0; i < length; ++i) dst[i] = (uint8_t)(src[i] + pred[i]); } else { - for (i = 0; i < length; ++i) dst[i] = src[i] - pred[i]; + for (i = 0; i < length; ++i) dst[i] = (uint8_t)(src[i] - pred[i]); } } @@ -155,7 +155,7 @@ static WEBP_INLINE void DoGradientFilter_C(const uint8_t* in, const int pred = GradientPredictor_C(preds[w - 1], preds[w - stride], preds[w - stride - 1]); - out[w] = in[w] + (inverse ? pred : -pred); + out[w] = (uint8_t)(in[w] + (inverse ? pred : -pred)); } ++row; preds += stride; @@ -194,7 +194,7 @@ static void HorizontalUnfilter_C(const uint8_t* prev, const uint8_t* in, uint8_t pred = (prev == NULL) ? 0 : prev[0]; int i; for (i = 0; i < width; ++i) { - out[i] = pred + in[i]; + out[i] = (uint8_t)(pred + in[i]); pred = out[i]; } } @@ -206,7 +206,7 @@ static void VerticalUnfilter_C(const uint8_t* prev, const uint8_t* in, HorizontalUnfilter_C(NULL, in, out, width); } else { int i; - for (i = 0; i < width; ++i) out[i] = prev[i] + in[i]; + for (i = 0; i < width; ++i) out[i] = (uint8_t)(prev[i] + in[i]); } } #endif // !WEBP_NEON_OMIT_C_CODE @@ -220,7 +220,7 @@ static void GradientUnfilter_C(const uint8_t* prev, const uint8_t* in, int i; for (i = 0; i < width; ++i) { top = prev[i]; // need to read this first, in case prev==out - left = in[i] + GradientPredictor_C(left, top, top_left); + left = (uint8_t)(in[i] + GradientPredictor_C(left, top, top_left)); top_left = top; out[i] = left; } diff --git a/thirdparty/libwebp/src/dsp/filters_sse2.c b/thirdparty/libwebp/src/dsp/filters_sse2.c index 5a188956761..4b3f2d020f4 100644 --- a/thirdparty/libwebp/src/dsp/filters_sse2.c +++ b/thirdparty/libwebp/src/dsp/filters_sse2.c @@ -163,7 +163,8 @@ static void GradientPredictDirect_SSE2(const uint8_t* const row, _mm_storel_epi64((__m128i*)(out + i), H); } for (; i < length; ++i) { - out[i] = row[i] - GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); + const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); + out[i] = (uint8_t)(row[i] - delta); } } @@ -188,7 +189,7 @@ static WEBP_INLINE void DoGradientFilter_SSE2(const uint8_t* in, // Filter line-by-line. while (row < last_row) { - out[0] = in[0] - in[-stride]; + out[0] = (uint8_t)(in[0] - in[-stride]); GradientPredictDirect_SSE2(in + 1, in + 1 - stride, out + 1, width - 1); ++row; in += stride; @@ -223,7 +224,7 @@ static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, uint8_t* out, int width) { int i; __m128i last; - out[0] = in[0] + (prev == NULL ? 0 : prev[0]); + out[0] = (uint8_t)(in[0] + (prev == NULL ? 0 : prev[0])); if (width <= 1) return; last = _mm_set_epi32(0, 0, 0, out[0]); for (i = 1; i + 8 <= width; i += 8) { @@ -238,7 +239,7 @@ static void HorizontalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, _mm_storel_epi64((__m128i*)(out + i), A7); last = _mm_srli_epi64(A7, 56); } - for (; i < width; ++i) out[i] = in[i] + out[i - 1]; + for (; i < width; ++i) out[i] = (uint8_t)(in[i] + out[i - 1]); } static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, @@ -259,7 +260,7 @@ static void VerticalUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, _mm_storeu_si128((__m128i*)&out[i + 0], C0); _mm_storeu_si128((__m128i*)&out[i + 16], C1); } - for (; i < width; ++i) out[i] = in[i] + prev[i]; + for (; i < width; ++i) out[i] = (uint8_t)(in[i] + prev[i]); } } @@ -296,7 +297,8 @@ static void GradientPredictInverse_SSE2(const uint8_t* const in, _mm_storel_epi64((__m128i*)&row[i], out); } for (; i < length; ++i) { - row[i] = in[i] + GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); + const int delta = GradientPredictor_SSE2(row[i - 1], top[i], top[i - 1]); + row[i] = (uint8_t)(in[i] + delta); } } } @@ -306,7 +308,7 @@ static void GradientUnfilter_SSE2(const uint8_t* prev, const uint8_t* in, if (prev == NULL) { HorizontalUnfilter_SSE2(NULL, in, out, width); } else { - out[0] = in[0] + prev[0]; // predict from above + out[0] = (uint8_t)(in[0] + prev[0]); // predict from above GradientPredictInverse_SSE2(in + 1, prev + 1, out + 1, width - 1); } } diff --git a/thirdparty/libwebp/src/dsp/lossless.c b/thirdparty/libwebp/src/dsp/lossless.c index d21aa6a0a0c..d05af84e7bc 100644 --- a/thirdparty/libwebp/src/dsp/lossless.c +++ b/thirdparty/libwebp/src/dsp/lossless.c @@ -270,14 +270,14 @@ void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, int i; for (i = 0; i < num_pixels; ++i) { const uint32_t argb = src[i]; - const uint32_t green = argb >> 8; + const int8_t green = (int8_t)(argb >> 8); const uint32_t red = argb >> 16; int new_red = red & 0xff; int new_blue = argb & 0xff; new_red += ColorTransformDelta(m->green_to_red_, green); new_red &= 0xff; new_blue += ColorTransformDelta(m->green_to_blue_, green); - new_blue += ColorTransformDelta(m->red_to_blue_, new_red); + new_blue += ColorTransformDelta(m->red_to_blue_, (int8_t)new_red); new_blue &= 0xff; dst[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } diff --git a/thirdparty/libwebp/src/dsp/lossless_enc.c b/thirdparty/libwebp/src/dsp/lossless_enc.c index 1408fbf580e..9c36055afc8 100644 --- a/thirdparty/libwebp/src/dsp/lossless_enc.c +++ b/thirdparty/libwebp/src/dsp/lossless_enc.c @@ -515,13 +515,17 @@ static WEBP_INLINE int ColorTransformDelta(int8_t color_pred, int8_t color) { return ((int)color_pred * color) >> 5; } +static WEBP_INLINE int8_t U32ToS8(uint32_t v) { + return (int8_t)(v & 0xff); +} + void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { const uint32_t argb = data[i]; - const uint32_t green = argb >> 8; - const uint32_t red = argb >> 16; + const int8_t green = U32ToS8(argb >> 8); + const int8_t red = U32ToS8(argb >> 16); int new_red = red & 0xff; int new_blue = argb & 0xff; new_red -= ColorTransformDelta(m->green_to_red_, green); @@ -535,7 +539,7 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, uint32_t argb) { - const uint32_t green = argb >> 8; + const int8_t green = U32ToS8(argb >> 8); int new_red = argb >> 16; new_red -= ColorTransformDelta(green_to_red, green); return (new_red & 0xff); @@ -544,9 +548,9 @@ static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, uint8_t red_to_blue, uint32_t argb) { - const uint32_t green = argb >> 8; - const uint32_t red = argb >> 16; - uint8_t new_blue = argb; + const int8_t green = U32ToS8(argb >> 8); + const int8_t red = U32ToS8(argb >> 16); + uint8_t new_blue = argb & 0xff; new_blue -= ColorTransformDelta(green_to_blue, green); new_blue -= ColorTransformDelta(red_to_blue, red); return (new_blue & 0xff); @@ -558,7 +562,7 @@ void VP8LCollectColorRedTransforms_C(const uint32_t* argb, int stride, while (tile_height-- > 0) { int x; for (x = 0; x < tile_width; ++x) { - ++histo[TransformColorRed(green_to_red, argb[x])]; + ++histo[TransformColorRed((uint8_t)green_to_red, argb[x])]; } argb += stride; } @@ -571,7 +575,8 @@ void VP8LCollectColorBlueTransforms_C(const uint32_t* argb, int stride, while (tile_height-- > 0) { int x; for (x = 0; x < tile_width; ++x) { - ++histo[TransformColorBlue(green_to_blue, red_to_blue, argb[x])]; + ++histo[TransformColorBlue((uint8_t)green_to_blue, (uint8_t)red_to_blue, + argb[x])]; } argb += stride; } diff --git a/thirdparty/libwebp/src/dsp/lossless_enc_sse2.c b/thirdparty/libwebp/src/dsp/lossless_enc_sse2.c index 36478c4912d..8adc52139bc 100644 --- a/thirdparty/libwebp/src/dsp/lossless_enc_sse2.c +++ b/thirdparty/libwebp/src/dsp/lossless_enc_sse2.c @@ -363,7 +363,7 @@ static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, assert(xbits <= 3); switch (xbits) { case 0: { - const __m128i ff = _mm_set1_epi16(0xff00); + const __m128i ff = _mm_set1_epi16((short)0xff00); const __m128i zero = _mm_setzero_si128(); // Store 0xff000000 | (row[x] << 8). for (x = 0; x + 16 <= width; x += 16, dst += 16) { @@ -382,7 +382,7 @@ static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, break; } case 1: { - const __m128i ff = _mm_set1_epi16(0xff00); + const __m128i ff = _mm_set1_epi16((short)0xff00); const __m128i mul = _mm_set1_epi16(0x110); for (x = 0; x + 16 <= width; x += 16, dst += 8) { // 0a0b | (where a/b are 4 bits). diff --git a/thirdparty/libwebp/src/dsp/lossless_enc_sse41.c b/thirdparty/libwebp/src/dsp/lossless_enc_sse41.c index 2e12a712eb4..719d8ed25e1 100644 --- a/thirdparty/libwebp/src/dsp/lossless_enc_sse41.c +++ b/thirdparty/libwebp/src/dsp/lossless_enc_sse41.c @@ -51,9 +51,9 @@ static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride, int histo[]) { const __m128i mults_r = _mm_set1_epi16(CST_5b(red_to_blue)); const __m128i mults_g = _mm_set1_epi16(CST_5b(green_to_blue)); - const __m128i mask_g = _mm_set1_epi16(0xff00); // green mask - const __m128i mask_gb = _mm_set1_epi32(0xffff); // green/blue mask - const __m128i mask_b = _mm_set1_epi16(0x00ff); // blue mask + const __m128i mask_g = _mm_set1_epi16((short)0xff00); // green mask + const __m128i mask_gb = _mm_set1_epi32(0xffff); // green/blue mask + const __m128i mask_b = _mm_set1_epi16(0x00ff); // blue mask const __m128i shuffler_lo = _mm_setr_epi8(-1, 2, -1, 6, -1, 10, -1, 14, -1, -1, -1, -1, -1, -1, -1, -1); const __m128i shuffler_hi = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, diff --git a/thirdparty/libwebp/src/dsp/quant.h b/thirdparty/libwebp/src/dsp/quant.h index 5ba6f9c3775..5e8dba8d19e 100644 --- a/thirdparty/libwebp/src/dsp/quant.h +++ b/thirdparty/libwebp/src/dsp/quant.h @@ -10,6 +10,8 @@ #ifndef WEBP_DSP_QUANT_H_ #define WEBP_DSP_QUANT_H_ +#include + #include "src/dsp/dsp.h" #include "src/webp/types.h" @@ -67,4 +69,17 @@ static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks, #endif // defined(WEBP_USE_NEON) && !defined(WEBP_ANDROID_NEON) && // !defined(WEBP_HAVE_NEON_RTCD) +static WEBP_INLINE int IsFlatSource16(const uint8_t* src) { + const uint32_t v = src[0] * 0x01010101u; + int i; + for (i = 0; i < 16; ++i) { + if (memcmp(src + 0, &v, 4) || memcmp(src + 4, &v, 4) || + memcmp(src + 8, &v, 4) || memcmp(src + 12, &v, 4)) { + return 0; + } + src += BPS; + } + return 1; +} + #endif // WEBP_DSP_QUANT_H_ diff --git a/thirdparty/libwebp/src/dsp/rescaler.c b/thirdparty/libwebp/src/dsp/rescaler.c index 753f84fcf4e..c5a01e82df5 100644 --- a/thirdparty/libwebp/src/dsp/rescaler.c +++ b/thirdparty/libwebp/src/dsp/rescaler.c @@ -109,8 +109,7 @@ void WebPRescalerExportRowExpand_C(WebPRescaler* const wrk) { for (x_out = 0; x_out < x_out_max; ++x_out) { const uint32_t J = frow[x_out]; const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } else { const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); @@ -120,8 +119,7 @@ void WebPRescalerExportRowExpand_C(WebPRescaler* const wrk) { + (uint64_t)B * irow[x_out]; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } @@ -138,17 +136,15 @@ void WebPRescalerExportRowShrink_C(WebPRescaler* const wrk) { assert(!wrk->y_expand); if (yscale) { for (x_out = 0; x_out < x_out_max; ++x_out) { - const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale); - const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = frac; // new fractional start } } else { for (x_out = 0; x_out < x_out_max; ++x_out) { const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = 0; } } diff --git a/thirdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c b/thirdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c index ce9e64862e2..419b741fa51 100644 --- a/thirdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c +++ b/thirdparty/libwebp/src/dsp/rescaler_mips_dsp_r2.c @@ -107,10 +107,9 @@ static void ExportRowShrink_MIPSdspR2(WebPRescaler* const wrk) { ); } for (i = 0; i < (x_out_max & 0x3); ++i) { - const uint32_t frac = (uint32_t)MULT_FIX(*frow++, yscale); - const int v = (int)MULT_FIX_FLOOR(*irow - frac, wrk->fxy_scale); - assert(v >= 0 && v <= 255); - *dst++ = v; + const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(*frow++, yscale); + const int v = (int)MULT_FIX(*irow - frac, wrk->fxy_scale); + *dst++ = (v > 255) ? 255u : (uint8_t)v; *irow++ = frac; // new fractional start } } else { @@ -157,8 +156,7 @@ static void ExportRowShrink_MIPSdspR2(WebPRescaler* const wrk) { } for (i = 0; i < (x_out_max & 0x3); ++i) { const int v = (int)MULT_FIX_FLOOR(*irow, wrk->fxy_scale); - assert(v >= 0 && v <= 255); - *dst++ = v; + *dst++ = (v > 255) ? 255u : (uint8_t)v; *irow++ = 0; } } @@ -219,8 +217,7 @@ static void ExportRowExpand_MIPSdspR2(WebPRescaler* const wrk) { for (i = 0; i < (x_out_max & 0x3); ++i) { const uint32_t J = *frow++; const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - *dst++ = v; + *dst++ = (v > 255) ? 255u : (uint8_t)v; } } else { const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); @@ -291,8 +288,7 @@ static void ExportRowExpand_MIPSdspR2(WebPRescaler* const wrk) { + (uint64_t)B * *irow++; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - *dst++ = v; + *dst++ = (v > 255) ? 255u : (uint8_t)v; } } } diff --git a/thirdparty/libwebp/src/dsp/rescaler_msa.c b/thirdparty/libwebp/src/dsp/rescaler_msa.c index c5592548368..256dbdd4375 100644 --- a/thirdparty/libwebp/src/dsp/rescaler_msa.c +++ b/thirdparty/libwebp/src/dsp/rescaler_msa.c @@ -166,8 +166,7 @@ static WEBP_INLINE void ExportRowExpand_0(const uint32_t* frow, uint8_t* dst, for (x_out = 0; x_out < length; ++x_out) { const uint32_t J = frow[x_out]; const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } @@ -241,8 +240,7 @@ static WEBP_INLINE void ExportRowExpand_1(const uint32_t* frow, uint32_t* irow, + (uint64_t)B * irow[x_out]; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } @@ -342,10 +340,9 @@ static WEBP_INLINE void ExportRowShrink_0(const uint32_t* frow, uint32_t* irow, length -= 4; } for (x_out = 0; x_out < length; ++x_out) { - const uint32_t frac = (uint32_t)MULT_FIX(frow[x_out], yscale); - const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + const uint32_t frac = (uint32_t)MULT_FIX_FLOOR(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = frac; } } @@ -406,8 +403,7 @@ static WEBP_INLINE void ExportRowShrink_1(uint32_t* irow, uint8_t* dst, } for (x_out = 0; x_out < length; ++x_out) { const int v = (int)MULT_FIX(irow[x_out], wrk->fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = 0; } } diff --git a/thirdparty/libwebp/src/dsp/rescaler_neon.c b/thirdparty/libwebp/src/dsp/rescaler_neon.c index a553f06f79c..b976a852cfc 100644 --- a/thirdparty/libwebp/src/dsp/rescaler_neon.c +++ b/thirdparty/libwebp/src/dsp/rescaler_neon.c @@ -81,14 +81,13 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) { const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half); const uint16x4_t C0 = vmovn_u32(B0); const uint16x4_t C1 = vmovn_u32(B1); - const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1)); + const uint8x8_t D = vqmovn_u16(vcombine_u16(C0, C1)); vst1_u8(dst + x_out, D); } for (; x_out < x_out_max; ++x_out) { const uint32_t J = frow[x_out]; const int v = (int)MULT_FIX_C(J, fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } else { const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); @@ -102,7 +101,7 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) { const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half); const uint16x4_t E0 = vmovn_u32(D0); const uint16x4_t E1 = vmovn_u32(D1); - const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1)); + const uint8x8_t F = vqmovn_u16(vcombine_u16(E0, E1)); vst1_u8(dst + x_out, F); } for (; x_out < x_out_max; ++x_out) { @@ -110,8 +109,7 @@ static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) { + (uint64_t)B * irow[x_out]; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX_C(J, fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } @@ -135,23 +133,22 @@ static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) { for (x_out = 0; x_out < max_span; x_out += 8) { LOAD_32x8(frow + x_out, in0, in1); LOAD_32x8(irow + x_out, in2, in3); - const uint32x4_t A0 = MULT_FIX(in0, yscale_half); - const uint32x4_t A1 = MULT_FIX(in1, yscale_half); + const uint32x4_t A0 = MULT_FIX_FLOOR(in0, yscale_half); + const uint32x4_t A1 = MULT_FIX_FLOOR(in1, yscale_half); const uint32x4_t B0 = vqsubq_u32(in2, A0); const uint32x4_t B1 = vqsubq_u32(in3, A1); - const uint32x4_t C0 = MULT_FIX_FLOOR(B0, fxy_scale_half); - const uint32x4_t C1 = MULT_FIX_FLOOR(B1, fxy_scale_half); + const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half); + const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half); const uint16x4_t D0 = vmovn_u32(C0); const uint16x4_t D1 = vmovn_u32(C1); - const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1)); + const uint8x8_t E = vqmovn_u16(vcombine_u16(D0, D1)); vst1_u8(dst + x_out, E); STORE_32x8(A0, A1, irow + x_out); } for (; x_out < x_out_max; ++x_out) { - const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale); - const int v = (int)MULT_FIX_FLOOR_C(irow[x_out] - frac, fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + const uint32_t frac = (uint32_t)MULT_FIX_FLOOR_C(frow[x_out], yscale); + const int v = (int)MULT_FIX_C(irow[x_out] - frac, fxy_scale); + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = frac; // new fractional start } } else { @@ -161,14 +158,13 @@ static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) { const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half); const uint16x4_t B0 = vmovn_u32(A0); const uint16x4_t B1 = vmovn_u32(A1); - const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1)); + const uint8x8_t C = vqmovn_u16(vcombine_u16(B0, B1)); vst1_u8(dst + x_out, C); STORE_32x8(zero, zero, irow + x_out); } for (; x_out < x_out_max; ++x_out) { const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = 0; } } diff --git a/thirdparty/libwebp/src/dsp/rescaler_sse2.c b/thirdparty/libwebp/src/dsp/rescaler_sse2.c index f7461a452c8..d7effea16ea 100644 --- a/thirdparty/libwebp/src/dsp/rescaler_sse2.c +++ b/thirdparty/libwebp/src/dsp/rescaler_sse2.c @@ -225,35 +225,6 @@ static WEBP_INLINE void ProcessRow_SSE2(const __m128i* const A0, _mm_storel_epi64((__m128i*)dst, G); } -static WEBP_INLINE void ProcessRow_Floor_SSE2(const __m128i* const A0, - const __m128i* const A1, - const __m128i* const A2, - const __m128i* const A3, - const __m128i* const mult, - uint8_t* const dst) { - const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0); - const __m128i B0 = _mm_mul_epu32(*A0, *mult); - const __m128i B1 = _mm_mul_epu32(*A1, *mult); - const __m128i B2 = _mm_mul_epu32(*A2, *mult); - const __m128i B3 = _mm_mul_epu32(*A3, *mult); - const __m128i D0 = _mm_srli_epi64(B0, WEBP_RESCALER_RFIX); - const __m128i D1 = _mm_srli_epi64(B1, WEBP_RESCALER_RFIX); -#if (WEBP_RESCALER_RFIX < 32) - const __m128i D2 = - _mm_and_si128(_mm_slli_epi64(B2, 32 - WEBP_RESCALER_RFIX), mask); - const __m128i D3 = - _mm_and_si128(_mm_slli_epi64(B3, 32 - WEBP_RESCALER_RFIX), mask); -#else - const __m128i D2 = _mm_and_si128(B2, mask); - const __m128i D3 = _mm_and_si128(B3, mask); -#endif - const __m128i E0 = _mm_or_si128(D0, D2); - const __m128i E1 = _mm_or_si128(D1, D3); - const __m128i F = _mm_packs_epi32(E0, E1); - const __m128i G = _mm_packus_epi16(F, F); - _mm_storel_epi64((__m128i*)dst, G); -} - static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) { int x_out; uint8_t* const dst = wrk->dst; @@ -274,8 +245,7 @@ static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) { for (; x_out < x_out_max; ++x_out) { const uint32_t J = frow[x_out]; const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } else { const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub); @@ -308,8 +278,7 @@ static void RescalerExportRowExpand_SSE2(WebPRescaler* const wrk) { + (uint64_t)B * irow[x_out]; const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX); const int v = (int)MULT_FIX(J, wrk->fy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; } } } @@ -328,20 +297,15 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) { const int scale_xy = wrk->fxy_scale; const __m128i mult_xy = _mm_set_epi32(0, scale_xy, 0, scale_xy); const __m128i mult_y = _mm_set_epi32(0, yscale, 0, yscale); - const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); for (x_out = 0; x_out + 8 <= x_out_max; x_out += 8) { __m128i A0, A1, A2, A3, B0, B1, B2, B3; LoadDispatchAndMult_SSE2(irow + x_out, NULL, &A0, &A1, &A2, &A3); LoadDispatchAndMult_SSE2(frow + x_out, &mult_y, &B0, &B1, &B2, &B3); { - const __m128i C0 = _mm_add_epi64(B0, rounder); - const __m128i C1 = _mm_add_epi64(B1, rounder); - const __m128i C2 = _mm_add_epi64(B2, rounder); - const __m128i C3 = _mm_add_epi64(B3, rounder); - const __m128i D0 = _mm_srli_epi64(C0, WEBP_RESCALER_RFIX); // = frac - const __m128i D1 = _mm_srli_epi64(C1, WEBP_RESCALER_RFIX); - const __m128i D2 = _mm_srli_epi64(C2, WEBP_RESCALER_RFIX); - const __m128i D3 = _mm_srli_epi64(C3, WEBP_RESCALER_RFIX); + const __m128i D0 = _mm_srli_epi64(B0, WEBP_RESCALER_RFIX); // = frac + const __m128i D1 = _mm_srli_epi64(B1, WEBP_RESCALER_RFIX); + const __m128i D2 = _mm_srli_epi64(B2, WEBP_RESCALER_RFIX); + const __m128i D3 = _mm_srli_epi64(B3, WEBP_RESCALER_RFIX); const __m128i E0 = _mm_sub_epi64(A0, D0); // irow[x] - frac const __m128i E1 = _mm_sub_epi64(A1, D1); const __m128i E2 = _mm_sub_epi64(A2, D2); @@ -352,14 +316,13 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) { const __m128i G1 = _mm_or_si128(D1, F3); _mm_storeu_si128((__m128i*)(irow + x_out + 0), G0); _mm_storeu_si128((__m128i*)(irow + x_out + 4), G1); - ProcessRow_Floor_SSE2(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); + ProcessRow_SSE2(&E0, &E1, &E2, &E3, &mult_xy, dst + x_out); } } for (; x_out < x_out_max; ++x_out) { - const uint32_t frac = (int)MULT_FIX(frow[x_out], yscale); - const int v = (int)MULT_FIX_FLOOR(irow[x_out] - frac, wrk->fxy_scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + const uint32_t frac = (int)MULT_FIX_FLOOR(frow[x_out], yscale); + const int v = (int)MULT_FIX(irow[x_out] - frac, wrk->fxy_scale); + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = frac; // new fractional start } } else { @@ -375,8 +338,7 @@ static void RescalerExportRowShrink_SSE2(WebPRescaler* const wrk) { } for (; x_out < x_out_max; ++x_out) { const int v = (int)MULT_FIX(irow[x_out], scale); - assert(v >= 0 && v <= 255); - dst[x_out] = v; + dst[x_out] = (v > 255) ? 255u : (uint8_t)v; irow[x_out] = 0; } } diff --git a/thirdparty/libwebp/src/enc/backward_references_enc.c b/thirdparty/libwebp/src/enc/backward_references_enc.c index 3ab7b0ac7d0..d445b40fc52 100644 --- a/thirdparty/libwebp/src/enc/backward_references_enc.c +++ b/thirdparty/libwebp/src/enc/backward_references_enc.c @@ -191,13 +191,14 @@ void VP8LHashChainClear(VP8LHashChain* const p) { // ----------------------------------------------------------------------------- -#define HASH_MULTIPLIER_HI (0xc6a4a793ULL) -#define HASH_MULTIPLIER_LO (0x5bd1e996ULL) +static const uint32_t kHashMultiplierHi = 0xc6a4a793u; +static const uint32_t kHashMultiplierLo = 0x5bd1e996u; -static WEBP_INLINE uint32_t GetPixPairHash64(const uint32_t* const argb) { +static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE +uint32_t GetPixPairHash64(const uint32_t* const argb) { uint32_t key; - key = (argb[1] * HASH_MULTIPLIER_HI) & 0xffffffffu; - key += (argb[0] * HASH_MULTIPLIER_LO) & 0xffffffffu; + key = argb[1] * kHashMultiplierHi; + key += argb[0] * kHashMultiplierLo; key = key >> (32 - HASH_BITS); return key; } diff --git a/thirdparty/libwebp/src/enc/histogram_enc.c b/thirdparty/libwebp/src/enc/histogram_enc.c index 8ac6fa8e026..d89b98524ab 100644 --- a/thirdparty/libwebp/src/enc/histogram_enc.c +++ b/thirdparty/libwebp/src/enc/histogram_enc.c @@ -929,9 +929,8 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, } mappings = (int*) WebPSafeMalloc(*num_used, sizeof(*mappings)); - if (mappings == NULL || !HistoQueueInit(&histo_queue, kHistoQueueSize)) { - goto End; - } + if (mappings == NULL) return 0; + if (!HistoQueueInit(&histo_queue, kHistoQueueSize)) goto End; // Fill the initial mapping. for (j = 0, iter = 0; iter < image_histo->size; ++iter) { if (histograms[iter] == NULL) continue; diff --git a/thirdparty/libwebp/src/enc/predictor_enc.c b/thirdparty/libwebp/src/enc/predictor_enc.c index 802e89693e1..2e6762ea0dd 100644 --- a/thirdparty/libwebp/src/enc/predictor_enc.c +++ b/thirdparty/libwebp/src/enc/predictor_enc.c @@ -202,7 +202,7 @@ static uint32_t NearLossless(uint32_t value, uint32_t predict, } if ((value >> 24) == 0 || (value >> 24) == 0xff) { // Preserve transparency of fully transparent or fully opaque pixels. - a = NearLosslessDiff(value >> 24, predict >> 24); + a = NearLosslessDiff((value >> 24) & 0xff, (predict >> 24) & 0xff); } else { a = NearLosslessComponent(value >> 24, predict >> 24, 0xff, quantization); } @@ -215,12 +215,12 @@ static uint32_t NearLossless(uint32_t value, uint32_t predict, // The amount by which green has been adjusted during quantization. It is // subtracted from red and blue for compensation, to avoid accumulating two // quantization errors in them. - green_diff = NearLosslessDiff(new_green, value >> 8); + green_diff = NearLosslessDiff(new_green, (value >> 8) & 0xff); } - r = NearLosslessComponent(NearLosslessDiff(value >> 16, green_diff), + r = NearLosslessComponent(NearLosslessDiff((value >> 16) & 0xff, green_diff), (predict >> 16) & 0xff, 0xff - new_green, quantization); - b = NearLosslessComponent(NearLosslessDiff(value, green_diff), + b = NearLosslessComponent(NearLosslessDiff(value & 0xff, green_diff), predict & 0xff, 0xff - new_green, quantization); return ((uint32_t)a << 24) | ((uint32_t)r << 16) | ((uint32_t)g << 8) | b; } @@ -587,7 +587,7 @@ static void GetBestGreenToRed( } } } - best_tx->green_to_red_ = green_to_red_best; + best_tx->green_to_red_ = (green_to_red_best & 0xff); } static float GetPredictionCostCrossColorBlue( @@ -666,8 +666,8 @@ static void GetBestGreenRedToBlue( break; // out of iter-loop. } } - best_tx->green_to_blue_ = green_to_blue_best; - best_tx->red_to_blue_ = red_to_blue_best; + best_tx->green_to_blue_ = green_to_blue_best & 0xff; + best_tx->red_to_blue_ = red_to_blue_best & 0xff; } #undef kGreenRedToBlueMaxIters #undef kGreenRedToBlueNumAxis diff --git a/thirdparty/libwebp/src/enc/quant_enc.c b/thirdparty/libwebp/src/enc/quant_enc.c index 03c682e3ae7..01eb565c7f9 100644 --- a/thirdparty/libwebp/src/enc/quant_enc.c +++ b/thirdparty/libwebp/src/enc/quant_enc.c @@ -33,7 +33,7 @@ // number of non-zero coeffs below which we consider the block very flat // (and apply a penalty to complex predictions) -#define FLATNESS_LIMIT_I16 10 // I16 mode +#define FLATNESS_LIMIT_I16 0 // I16 mode (special case) #define FLATNESS_LIMIT_I4 3 // I4 mode #define FLATNESS_LIMIT_UV 2 // UV mode #define FLATNESS_PENALTY 140 // roughly ~1bit per block @@ -988,6 +988,7 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { VP8ModeScore* rd_cur = &rd_tmp; VP8ModeScore* rd_best = rd; int mode; + int is_flat = IsFlatSource16(it->yuv_in_ + Y_OFF_ENC); rd->mode_i16 = -1; for (mode = 0; mode < NUM_PRED_MODES; ++mode) { @@ -1003,10 +1004,14 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { tlambda ? MULT_8B(tlambda, VP8TDisto16x16(src, tmp_dst, kWeightY)) : 0; rd_cur->H = VP8FixedCostsI16[mode]; rd_cur->R = VP8GetCostLuma16(it, rd_cur); - if (mode > 0 && - IsFlat(rd_cur->y_ac_levels[0], kNumBlocks, FLATNESS_LIMIT_I16)) { - // penalty to avoid flat area to be mispredicted by complex mode - rd_cur->R += FLATNESS_PENALTY * kNumBlocks; + if (is_flat) { + // refine the first impression (which was in pixel space) + is_flat = IsFlat(rd_cur->y_ac_levels[0], kNumBlocks, FLATNESS_LIMIT_I16); + if (is_flat) { + // Block is very flat. We put emphasis on the distortion being very low! + rd_cur->D *= 2; + rd_cur->SD *= 2; + } } // Since we always examine Intra16 first, we can overwrite *rd directly. @@ -1087,7 +1092,8 @@ static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) { : 0; rd_tmp.H = mode_costs[mode]; - // Add flatness penalty + // Add flatness penalty, to avoid flat area to be mispredicted + // by a complex mode. if (mode > 0 && IsFlat(tmp_levels, kNumBlocks, FLATNESS_LIMIT_I4)) { rd_tmp.R = FLATNESS_PENALTY * kNumBlocks; } else { @@ -1242,11 +1248,19 @@ static void RefineUsingDistortion(VP8EncIterator* const it, if (mode > 0 && VP8FixedCostsI16[mode] > bit_limit) { continue; } + if (score < best_score) { best_mode = mode; best_score = score; } } + if (it->x_ == 0 || it->y_ == 0) { + // avoid starting a checkerboard resonance from the border. See bug #432. + if (IsFlatSource16(src)) { + best_mode = (it->x_ == 0) ? 0 : 2; + try_both_modes = 0; // stick to i16 + } + } VP8SetIntra16Mode(it, best_mode); // we'll reconstruct later, if i16 mode actually gets selected } diff --git a/thirdparty/libwebp/src/enc/vp8i_enc.h b/thirdparty/libwebp/src/enc/vp8i_enc.h index 3a1967da881..24e1944610c 100644 --- a/thirdparty/libwebp/src/enc/vp8i_enc.h +++ b/thirdparty/libwebp/src/enc/vp8i_enc.h @@ -32,7 +32,7 @@ extern "C" { // version numbers #define ENC_MAJ_VERSION 1 #define ENC_MIN_VERSION 0 -#define ENC_REV_VERSION 2 +#define ENC_REV_VERSION 3 enum { MAX_LF_LEVELS = 64, // Maximum loop filter level MAX_VARIABLE_LEVEL = 67, // last (inclusive) level with variable cost diff --git a/thirdparty/libwebp/src/mux/muxi.h b/thirdparty/libwebp/src/mux/muxi.h index 3e9d8c48d81..7bc0b07e9ba 100644 --- a/thirdparty/libwebp/src/mux/muxi.h +++ b/thirdparty/libwebp/src/mux/muxi.h @@ -29,7 +29,7 @@ extern "C" { #define MUX_MAJ_VERSION 1 #define MUX_MIN_VERSION 0 -#define MUX_REV_VERSION 2 +#define MUX_REV_VERSION 3 // Chunk object. typedef struct WebPChunk WebPChunk; diff --git a/thirdparty/libwebp/src/utils/bit_reader_inl_utils.h b/thirdparty/libwebp/src/utils/bit_reader_inl_utils.h index 7e607f370a0..46b38807062 100644 --- a/thirdparty/libwebp/src/utils/bit_reader_inl_utils.h +++ b/thirdparty/libwebp/src/utils/bit_reader_inl_utils.h @@ -104,7 +104,8 @@ void VP8LoadNewBytes(VP8BitReader* const br) { } // Read a bit with proba 'prob'. Speed-critical function! -static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) { +static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, + int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't // alter br->range_ value. @@ -129,13 +130,14 @@ static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) { br->bits_ -= shift; } br->range_ = range - 1; + BT_TRACK(br); return bit; } } // simplified version of VP8GetBit() for prob=0x80 (note shift is always 1 here) static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE -int VP8GetSigned(VP8BitReader* const br, int v) { +int VP8GetSigned(VP8BitReader* const br, int v, const char label[]) { if (br->bits_ < 0) { VP8LoadNewBytes(br); } @@ -148,11 +150,13 @@ int VP8GetSigned(VP8BitReader* const br, int v) { br->range_ += mask; br->range_ |= 1; br->value_ -= (bit_t)((split + 1) & mask) << pos; + BT_TRACK(br); return (v ^ mask) - mask; } } -static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* const br, int prob) { +static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* const br, + int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't // alter br->range_ value. @@ -179,6 +183,7 @@ static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* const br, int prob) { br->bits_ -= shift; } br->range_ = range; + BT_TRACK(br); return bit; } } diff --git a/thirdparty/libwebp/src/utils/bit_reader_utils.c b/thirdparty/libwebp/src/utils/bit_reader_utils.c index 5fa3ae77955..60271c0ae06 100644 --- a/thirdparty/libwebp/src/utils/bit_reader_utils.c +++ b/thirdparty/libwebp/src/utils/bit_reader_utils.c @@ -109,17 +109,18 @@ void VP8LoadFinalBytes(VP8BitReader* const br) { //------------------------------------------------------------------------------ // Higher-level calls -uint32_t VP8GetValue(VP8BitReader* const br, int bits) { +uint32_t VP8GetValue(VP8BitReader* const br, int bits, const char label[]) { uint32_t v = 0; while (bits-- > 0) { - v |= VP8GetBit(br, 0x80) << bits; + v |= VP8GetBit(br, 0x80, label) << bits; } return v; } -int32_t VP8GetSignedValue(VP8BitReader* const br, int bits) { - const int value = VP8GetValue(br, bits); - return VP8Get(br) ? -value : value; +int32_t VP8GetSignedValue(VP8BitReader* const br, int bits, + const char label[]) { + const int value = VP8GetValue(br, bits, label); + return VP8Get(br, label) ? -value : value; } //------------------------------------------------------------------------------ @@ -227,3 +228,78 @@ uint32_t VP8LReadBits(VP8LBitReader* const br, int n_bits) { } //------------------------------------------------------------------------------ +// Bit-tracing tool + +#if (BITTRACE > 0) + +#include // for atexit() +#include +#include + +#define MAX_NUM_LABELS 32 +static struct { + const char* label; + int size; + int count; +} kLabels[MAX_NUM_LABELS]; + +static int last_label = 0; +static int last_pos = 0; +static const uint8_t* buf_start = NULL; +static int init_done = 0; + +static void PrintBitTraces(void) { + int i; + int scale = 1; + int total = 0; + const char* units = "bits"; +#if (BITTRACE == 2) + scale = 8; + units = "bytes"; +#endif + for (i = 0; i < last_label; ++i) total += kLabels[i].size; + if (total < 1) total = 1; // avoid rounding errors + printf("=== Bit traces ===\n"); + for (i = 0; i < last_label; ++i) { + const int skip = 16 - (int)strlen(kLabels[i].label); + const int value = (kLabels[i].size + scale - 1) / scale; + assert(skip > 0); + printf("%s \%*s: %6d %s \t[%5.2f%%] [count: %7d]\n", + kLabels[i].label, skip, "", value, units, + 100.f * kLabels[i].size / total, + kLabels[i].count); + } + total = (total + scale - 1) / scale; + printf("Total: %d %s\n", total, units); +} + +void BitTrace(const struct VP8BitReader* const br, const char label[]) { + int i, pos; + if (!init_done) { + memset(kLabels, 0, sizeof(kLabels)); + atexit(PrintBitTraces); + buf_start = br->buf_; + init_done = 1; + } + pos = (int)(br->buf_ - buf_start) * 8 - br->bits_; + // if there's a too large jump, we've changed partition -> reset counter + if (abs(pos - last_pos) > 32) { + buf_start = br->buf_; + pos = 0; + last_pos = 0; + } + if (br->range_ >= 0x7f) pos += kVP8Log2Range[br->range_ - 0x7f]; + for (i = 0; i < last_label; ++i) { + if (!strcmp(label, kLabels[i].label)) break; + } + if (i == MAX_NUM_LABELS) abort(); // overflow! + kLabels[i].label = label; + kLabels[i].size += pos - last_pos; + kLabels[i].count += 1; + if (i == last_label) ++last_label; + last_pos = pos; +} + +#endif // BITTRACE > 0 + +//------------------------------------------------------------------------------ diff --git a/thirdparty/libwebp/src/utils/bit_reader_utils.h b/thirdparty/libwebp/src/utils/bit_reader_utils.h index de810d402ac..199dacf2241 100644 --- a/thirdparty/libwebp/src/utils/bit_reader_utils.h +++ b/thirdparty/libwebp/src/utils/bit_reader_utils.h @@ -21,6 +21,27 @@ #endif #include "src/webp/types.h" +// Warning! This macro triggers quite some MACRO wizardry around func signature! +#if !defined(BITTRACE) +#define BITTRACE 0 // 0 = off, 1 = print bits, 2 = print bytes +#endif + +#if (BITTRACE > 0) +struct VP8BitReader; +extern void BitTrace(const struct VP8BitReader* const br, const char label[]); +#define BT_TRACK(br) BitTrace(br, label) +#define VP8Get(BR, L) VP8GetValue(BR, 1, L) +#else +#define BT_TRACK(br) +// We'll REMOVE the 'const char label[]' from all signatures and calls (!!): +#define VP8GetValue(BR, N, L) VP8GetValue(BR, N) +#define VP8Get(BR, L) VP8GetValue(BR, 1, L) +#define VP8GetSignedValue(BR, N, L) VP8GetSignedValue(BR, N) +#define VP8GetBit(BR, P, L) VP8GetBit(BR, P) +#define VP8GetBitAlt(BR, P, L) VP8GetBitAlt(BR, P) +#define VP8GetSigned(BR, V, L) VP8GetSigned(BR, V) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -102,17 +123,15 @@ void VP8BitReaderSetBuffer(VP8BitReader* const br, void VP8RemapBitReader(VP8BitReader* const br, ptrdiff_t offset); // return the next value made of 'num_bits' bits -uint32_t VP8GetValue(VP8BitReader* const br, int num_bits); -static WEBP_INLINE uint32_t VP8Get(VP8BitReader* const br) { - return VP8GetValue(br, 1); -} +uint32_t VP8GetValue(VP8BitReader* const br, int num_bits, const char label[]); // return the next value with sign-extension. -int32_t VP8GetSignedValue(VP8BitReader* const br, int num_bits); +int32_t VP8GetSignedValue(VP8BitReader* const br, int num_bits, + const char label[]); // bit_reader_inl.h will implement the following methods: -// static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob) -// static WEBP_INLINE int VP8GetSigned(VP8BitReader* const br, int v) +// static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, int prob, ...) +// static WEBP_INLINE int VP8GetSigned(VP8BitReader* const br, int v, ...) // and should be included by the .c files that actually need them. // This is to avoid recompiling the whole library whenever this file is touched, // and also allowing platform-specific ad-hoc hacks. diff --git a/thirdparty/libwebp/src/utils/bit_writer_utils.c b/thirdparty/libwebp/src/utils/bit_writer_utils.c index 7f83b4c8a2b..bef0e31ca5e 100644 --- a/thirdparty/libwebp/src/utils/bit_writer_utils.c +++ b/thirdparty/libwebp/src/utils/bit_writer_utils.c @@ -70,7 +70,7 @@ static void Flush(VP8BitWriter* const bw) { const int value = (bits & 0x100) ? 0x00 : 0xff; for (; bw->run_ > 0; --bw->run_) bw->buf_[pos++] = value; } - bw->buf_[pos++] = bits; + bw->buf_[pos++] = bits & 0xff; bw->pos_ = pos; } else { bw->run_++; // delay writing of bytes 0xff, pending eventual carry. diff --git a/thirdparty/libwebp/src/utils/color_cache_utils.h b/thirdparty/libwebp/src/utils/color_cache_utils.h index 20b7be11c92..ec21d5199b1 100644 --- a/thirdparty/libwebp/src/utils/color_cache_utils.h +++ b/thirdparty/libwebp/src/utils/color_cache_utils.h @@ -17,6 +17,7 @@ #include +#include "src/dsp/dsp.h" #include "src/webp/types.h" #ifdef __cplusplus @@ -30,10 +31,11 @@ typedef struct { int hash_bits_; } VP8LColorCache; -static const uint64_t kHashMul = 0x1e35a7bdull; +static const uint32_t kHashMul = 0x1e35a7bdu; -static WEBP_INLINE int VP8LHashPix(uint32_t argb, int shift) { - return (int)(((argb * kHashMul) & 0xffffffffu) >> shift); +static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE +int VP8LHashPix(uint32_t argb, int shift) { + return (int)((argb * kHashMul) >> shift); } static WEBP_INLINE uint32_t VP8LColorCacheLookup( diff --git a/thirdparty/libwebp/src/utils/huffman_utils.c b/thirdparty/libwebp/src/utils/huffman_utils.c index 7a69963c3e9..0cba0fbb7d4 100644 --- a/thirdparty/libwebp/src/utils/huffman_utils.c +++ b/thirdparty/libwebp/src/utils/huffman_utils.c @@ -91,7 +91,8 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, assert(code_lengths_size != 0); assert(code_lengths != NULL); - assert(root_table != NULL); + assert((root_table != NULL && sorted != NULL) || + (root_table == NULL && sorted == NULL)); assert(root_bits > 0); // Build histogram of code lengths. @@ -120,16 +121,22 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, for (symbol = 0; symbol < code_lengths_size; ++symbol) { const int symbol_code_length = code_lengths[symbol]; if (code_lengths[symbol] > 0) { - sorted[offset[symbol_code_length]++] = symbol; + if (sorted != NULL) { + sorted[offset[symbol_code_length]++] = symbol; + } else { + offset[symbol_code_length]++; + } } } // Special case code with only one value. if (offset[MAX_ALLOWED_CODE_LENGTH] == 1) { - HuffmanCode code; - code.bits = 0; - code.value = (uint16_t)sorted[0]; - ReplicateValue(table, 1, total_size, code); + if (sorted != NULL) { + HuffmanCode code; + code.bits = 0; + code.value = (uint16_t)sorted[0]; + ReplicateValue(table, 1, total_size, code); + } return total_size; } @@ -151,6 +158,7 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, if (num_open < 0) { return 0; } + if (root_table == NULL) continue; for (; count[len] > 0; --count[len]) { HuffmanCode code; code.bits = (uint8_t)len; @@ -169,6 +177,7 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, if (num_open < 0) { return 0; } + if (root_table == NULL) continue; for (; count[len] > 0; --count[len]) { HuffmanCode code; if ((key & mask) != low) { @@ -206,7 +215,10 @@ int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, const int code_lengths[], int code_lengths_size) { int total_size; assert(code_lengths_size <= MAX_CODE_LENGTHS_SIZE); - if (code_lengths_size <= SORTED_SIZE_CUTOFF) { + if (root_table == NULL) { + total_size = BuildHuffmanTable(NULL, root_bits, + code_lengths, code_lengths_size, NULL); + } else if (code_lengths_size <= SORTED_SIZE_CUTOFF) { // use local stack-allocated array. uint16_t sorted[SORTED_SIZE_CUTOFF]; total_size = BuildHuffmanTable(root_table, root_bits, diff --git a/thirdparty/libwebp/src/utils/huffman_utils.h b/thirdparty/libwebp/src/utils/huffman_utils.h index ff7ef17f3b8..13b7ad1ac40 100644 --- a/thirdparty/libwebp/src/utils/huffman_utils.h +++ b/thirdparty/libwebp/src/utils/huffman_utils.h @@ -78,6 +78,8 @@ void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups); // the huffman table. // Returns built table size or 0 in case of error (invalid tree or // memory error). +// If root_table is NULL, it returns 0 if a lookup cannot be built, something +// > 0 otherwise (but not the table size). int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, const int code_lengths[], int code_lengths_size); diff --git a/thirdparty/libwebp/src/utils/rescaler_utils.c b/thirdparty/libwebp/src/utils/rescaler_utils.c index 90e2ea76a1c..4bcae24af54 100644 --- a/thirdparty/libwebp/src/utils/rescaler_utils.c +++ b/thirdparty/libwebp/src/utils/rescaler_utils.c @@ -84,14 +84,14 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, int height = *scaled_height; // if width is unspecified, scale original proportionally to height ratio. - if (width == 0) { + if (width == 0 && src_height > 0) { width = - (int)(((uint64_t)src_width * height + src_height / 2) / src_height); + (int)(((uint64_t)src_width * height + src_height - 1) / src_height); } // if height is unspecified, scale original proportionally to width ratio. - if (height == 0) { + if (height == 0 && src_width > 0) { height = - (int)(((uint64_t)src_height * width + src_width / 2) / src_width); + (int)(((uint64_t)src_height * width + src_width - 1) / src_width); } // Check if the overall dimensions still make sense. if (width <= 0 || height <= 0) { diff --git a/thirdparty/libwebp/src/utils/thread_utils.c b/thirdparty/libwebp/src/utils/thread_utils.c index 2052b6b0069..438296b45f3 100644 --- a/thirdparty/libwebp/src/utils/thread_utils.c +++ b/thirdparty/libwebp/src/utils/thread_utils.c @@ -217,8 +217,12 @@ static THREADFN ThreadLoop(void* ptr) { done = 1; } // signal to the main thread that we're done (for Sync()) - pthread_cond_signal(&impl->condition_); + // Note the associated mutex does not need to be held when signaling the + // condition. Unlocking the mutex first may improve performance in some + // implementations, avoiding the case where the waiting thread can't + // reacquire the mutex when woken. pthread_mutex_unlock(&impl->mutex_); + pthread_cond_signal(&impl->condition_); } return THREAD_RETURN(NULL); // Thread is finished } @@ -240,7 +244,13 @@ static void ChangeState(WebPWorker* const worker, WebPWorkerStatus new_status) { // assign new status and release the working thread if needed if (new_status != OK) { worker->status_ = new_status; + // Note the associated mutex does not need to be held when signaling the + // condition. Unlocking the mutex first may improve performance in some + // implementations, avoiding the case where the waiting thread can't + // reacquire the mutex when woken. + pthread_mutex_unlock(&impl->mutex_); pthread_cond_signal(&impl->condition_); + return; } } pthread_mutex_unlock(&impl->mutex_); diff --git a/thirdparty/libwebp/src/utils/utils.h b/thirdparty/libwebp/src/utils/utils.h index c7620f91ec7..2a3ec926784 100644 --- a/thirdparty/libwebp/src/utils/utils.h +++ b/thirdparty/libwebp/src/utils/utils.h @@ -92,14 +92,14 @@ static WEBP_INLINE uint32_t GetLE32(const uint8_t* const data) { // Store 16, 24 or 32 bits in little-endian order. static WEBP_INLINE void PutLE16(uint8_t* const data, int val) { assert(val < (1 << 16)); - data[0] = (val >> 0); - data[1] = (val >> 8); + data[0] = (val >> 0) & 0xff; + data[1] = (val >> 8) & 0xff; } static WEBP_INLINE void PutLE24(uint8_t* const data, int val) { assert(val < (1 << 24)); PutLE16(data, val & 0xffff); - data[2] = (val >> 16); + data[2] = (val >> 16) & 0xff; } static WEBP_INLINE void PutLE32(uint8_t* const data, uint32_t val) { diff --git a/thirdparty/libwebp/src/webp/encode.h b/thirdparty/libwebp/src/webp/encode.h index 549cf077306..339f8810aa1 100644 --- a/thirdparty/libwebp/src/webp/encode.h +++ b/thirdparty/libwebp/src/webp/encode.h @@ -62,6 +62,10 @@ WEBP_EXTERN size_t WebPEncodeBGRA(const uint8_t* bgra, // These functions are the equivalent of the above, but compressing in a // lossless manner. Files are usually larger than lossy format, but will // not suffer any compression loss. +// Note these functions, like the lossy versions, use the library's default +// settings. For lossless this means 'exact' is disabled. RGB values in +// transparent areas will be modified to improve compression. To avoid this, +// use WebPEncode() and set WebPConfig::exact to 1. WEBP_EXTERN size_t WebPEncodeLosslessRGB(const uint8_t* rgb, int width, int height, int stride, uint8_t** output); From 8851026e8a227e3cff8cfb7001bad31d780c0015 Mon Sep 17 00:00:00 2001 From: Jonathan Mannancheril Date: Wed, 30 Oct 2019 17:40:59 -0500 Subject: [PATCH 014/113] Update libogg to upstream c8fca6b (cherry picked from commits 6eb8c11d3244c0a36fee4b09122e10e14ea7565e and e70e89443324abb93e47cf0d5a516e350b619097) --- thirdparty/README.md | 4 +- thirdparty/libogg/bitwise.c | 5 +- thirdparty/libogg/crctable.h | 278 +++++++++++++++++++++++++++ thirdparty/libogg/framing.c | 171 +++++++--------- thirdparty/libogg/ogg/config_types.h | 1 + thirdparty/libogg/ogg/ogg.h | 1 - thirdparty/libogg/ogg/os_types.h | 20 +- 7 files changed, 368 insertions(+), 112 deletions(-) create mode 100644 thirdparty/libogg/crctable.h diff --git a/thirdparty/README.md b/thirdparty/README.md index 561b6a2b436..ad703bcdff6 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -153,12 +153,12 @@ Files extracted from upstream source: ## libogg - Upstream: https://www.xiph.org/ogg -- Version: 1.3.3 +- Version: git (c8fca6b, 2019) - License: BSD-3-Clause Files extracted from upstream source: -- `src/*.c` +- `src/*.{c,h}` - `include/ogg/*.h` in ogg/ - COPYING diff --git a/thirdparty/libogg/bitwise.c b/thirdparty/libogg/bitwise.c index fa2b572029e..f5ef79122eb 100644 --- a/thirdparty/libogg/bitwise.c +++ b/thirdparty/libogg/bitwise.c @@ -11,7 +11,6 @@ ******************************************************************** function: packing variable sized words into an octet stream - last mod: $Id$ ********************************************************************/ @@ -890,7 +889,7 @@ int main(void){ for(i=0;i + +static const ogg_uint32_t crc_lookup[8][256]={ +{0x00000000,0x04c11db7,0x09823b6e,0x0d4326d9,0x130476dc,0x17c56b6b,0x1a864db2,0x1e475005, + 0x2608edb8,0x22c9f00f,0x2f8ad6d6,0x2b4bcb61,0x350c9b64,0x31cd86d3,0x3c8ea00a,0x384fbdbd, + 0x4c11db70,0x48d0c6c7,0x4593e01e,0x4152fda9,0x5f15adac,0x5bd4b01b,0x569796c2,0x52568b75, + 0x6a1936c8,0x6ed82b7f,0x639b0da6,0x675a1011,0x791d4014,0x7ddc5da3,0x709f7b7a,0x745e66cd, + 0x9823b6e0,0x9ce2ab57,0x91a18d8e,0x95609039,0x8b27c03c,0x8fe6dd8b,0x82a5fb52,0x8664e6e5, + 0xbe2b5b58,0xbaea46ef,0xb7a96036,0xb3687d81,0xad2f2d84,0xa9ee3033,0xa4ad16ea,0xa06c0b5d, + 0xd4326d90,0xd0f37027,0xddb056fe,0xd9714b49,0xc7361b4c,0xc3f706fb,0xceb42022,0xca753d95, + 0xf23a8028,0xf6fb9d9f,0xfbb8bb46,0xff79a6f1,0xe13ef6f4,0xe5ffeb43,0xe8bccd9a,0xec7dd02d, + 0x34867077,0x30476dc0,0x3d044b19,0x39c556ae,0x278206ab,0x23431b1c,0x2e003dc5,0x2ac12072, + 0x128e9dcf,0x164f8078,0x1b0ca6a1,0x1fcdbb16,0x018aeb13,0x054bf6a4,0x0808d07d,0x0cc9cdca, + 0x7897ab07,0x7c56b6b0,0x71159069,0x75d48dde,0x6b93dddb,0x6f52c06c,0x6211e6b5,0x66d0fb02, + 0x5e9f46bf,0x5a5e5b08,0x571d7dd1,0x53dc6066,0x4d9b3063,0x495a2dd4,0x44190b0d,0x40d816ba, + 0xaca5c697,0xa864db20,0xa527fdf9,0xa1e6e04e,0xbfa1b04b,0xbb60adfc,0xb6238b25,0xb2e29692, + 0x8aad2b2f,0x8e6c3698,0x832f1041,0x87ee0df6,0x99a95df3,0x9d684044,0x902b669d,0x94ea7b2a, + 0xe0b41de7,0xe4750050,0xe9362689,0xedf73b3e,0xf3b06b3b,0xf771768c,0xfa325055,0xfef34de2, + 0xc6bcf05f,0xc27dede8,0xcf3ecb31,0xcbffd686,0xd5b88683,0xd1799b34,0xdc3abded,0xd8fba05a, + 0x690ce0ee,0x6dcdfd59,0x608edb80,0x644fc637,0x7a089632,0x7ec98b85,0x738aad5c,0x774bb0eb, + 0x4f040d56,0x4bc510e1,0x46863638,0x42472b8f,0x5c007b8a,0x58c1663d,0x558240e4,0x51435d53, + 0x251d3b9e,0x21dc2629,0x2c9f00f0,0x285e1d47,0x36194d42,0x32d850f5,0x3f9b762c,0x3b5a6b9b, + 0x0315d626,0x07d4cb91,0x0a97ed48,0x0e56f0ff,0x1011a0fa,0x14d0bd4d,0x19939b94,0x1d528623, + 0xf12f560e,0xf5ee4bb9,0xf8ad6d60,0xfc6c70d7,0xe22b20d2,0xe6ea3d65,0xeba91bbc,0xef68060b, + 0xd727bbb6,0xd3e6a601,0xdea580d8,0xda649d6f,0xc423cd6a,0xc0e2d0dd,0xcda1f604,0xc960ebb3, + 0xbd3e8d7e,0xb9ff90c9,0xb4bcb610,0xb07daba7,0xae3afba2,0xaafbe615,0xa7b8c0cc,0xa379dd7b, + 0x9b3660c6,0x9ff77d71,0x92b45ba8,0x9675461f,0x8832161a,0x8cf30bad,0x81b02d74,0x857130c3, + 0x5d8a9099,0x594b8d2e,0x5408abf7,0x50c9b640,0x4e8ee645,0x4a4ffbf2,0x470cdd2b,0x43cdc09c, + 0x7b827d21,0x7f436096,0x7200464f,0x76c15bf8,0x68860bfd,0x6c47164a,0x61043093,0x65c52d24, + 0x119b4be9,0x155a565e,0x18197087,0x1cd86d30,0x029f3d35,0x065e2082,0x0b1d065b,0x0fdc1bec, + 0x3793a651,0x3352bbe6,0x3e119d3f,0x3ad08088,0x2497d08d,0x2056cd3a,0x2d15ebe3,0x29d4f654, + 0xc5a92679,0xc1683bce,0xcc2b1d17,0xc8ea00a0,0xd6ad50a5,0xd26c4d12,0xdf2f6bcb,0xdbee767c, + 0xe3a1cbc1,0xe760d676,0xea23f0af,0xeee2ed18,0xf0a5bd1d,0xf464a0aa,0xf9278673,0xfde69bc4, + 0x89b8fd09,0x8d79e0be,0x803ac667,0x84fbdbd0,0x9abc8bd5,0x9e7d9662,0x933eb0bb,0x97ffad0c, + 0xafb010b1,0xab710d06,0xa6322bdf,0xa2f33668,0xbcb4666d,0xb8757bda,0xb5365d03,0xb1f740b4}, + +{0x00000000,0xd219c1dc,0xa0f29e0f,0x72eb5fd3,0x452421a9,0x973de075,0xe5d6bfa6,0x37cf7e7a, + 0x8a484352,0x5851828e,0x2abadd5d,0xf8a31c81,0xcf6c62fb,0x1d75a327,0x6f9efcf4,0xbd873d28, + 0x10519b13,0xc2485acf,0xb0a3051c,0x62bac4c0,0x5575baba,0x876c7b66,0xf58724b5,0x279ee569, + 0x9a19d841,0x4800199d,0x3aeb464e,0xe8f28792,0xdf3df9e8,0x0d243834,0x7fcf67e7,0xadd6a63b, + 0x20a33626,0xf2baf7fa,0x8051a829,0x524869f5,0x6587178f,0xb79ed653,0xc5758980,0x176c485c, + 0xaaeb7574,0x78f2b4a8,0x0a19eb7b,0xd8002aa7,0xefcf54dd,0x3dd69501,0x4f3dcad2,0x9d240b0e, + 0x30f2ad35,0xe2eb6ce9,0x9000333a,0x4219f2e6,0x75d68c9c,0xa7cf4d40,0xd5241293,0x073dd34f, + 0xbabaee67,0x68a32fbb,0x1a487068,0xc851b1b4,0xff9ecfce,0x2d870e12,0x5f6c51c1,0x8d75901d, + 0x41466c4c,0x935fad90,0xe1b4f243,0x33ad339f,0x04624de5,0xd67b8c39,0xa490d3ea,0x76891236, + 0xcb0e2f1e,0x1917eec2,0x6bfcb111,0xb9e570cd,0x8e2a0eb7,0x5c33cf6b,0x2ed890b8,0xfcc15164, + 0x5117f75f,0x830e3683,0xf1e56950,0x23fca88c,0x1433d6f6,0xc62a172a,0xb4c148f9,0x66d88925, + 0xdb5fb40d,0x094675d1,0x7bad2a02,0xa9b4ebde,0x9e7b95a4,0x4c625478,0x3e890bab,0xec90ca77, + 0x61e55a6a,0xb3fc9bb6,0xc117c465,0x130e05b9,0x24c17bc3,0xf6d8ba1f,0x8433e5cc,0x562a2410, + 0xebad1938,0x39b4d8e4,0x4b5f8737,0x994646eb,0xae893891,0x7c90f94d,0x0e7ba69e,0xdc626742, + 0x71b4c179,0xa3ad00a5,0xd1465f76,0x035f9eaa,0x3490e0d0,0xe689210c,0x94627edf,0x467bbf03, + 0xfbfc822b,0x29e543f7,0x5b0e1c24,0x8917ddf8,0xbed8a382,0x6cc1625e,0x1e2a3d8d,0xcc33fc51, + 0x828cd898,0x50951944,0x227e4697,0xf067874b,0xc7a8f931,0x15b138ed,0x675a673e,0xb543a6e2, + 0x08c49bca,0xdadd5a16,0xa83605c5,0x7a2fc419,0x4de0ba63,0x9ff97bbf,0xed12246c,0x3f0be5b0, + 0x92dd438b,0x40c48257,0x322fdd84,0xe0361c58,0xd7f96222,0x05e0a3fe,0x770bfc2d,0xa5123df1, + 0x189500d9,0xca8cc105,0xb8679ed6,0x6a7e5f0a,0x5db12170,0x8fa8e0ac,0xfd43bf7f,0x2f5a7ea3, + 0xa22feebe,0x70362f62,0x02dd70b1,0xd0c4b16d,0xe70bcf17,0x35120ecb,0x47f95118,0x95e090c4, + 0x2867adec,0xfa7e6c30,0x889533e3,0x5a8cf23f,0x6d438c45,0xbf5a4d99,0xcdb1124a,0x1fa8d396, + 0xb27e75ad,0x6067b471,0x128ceba2,0xc0952a7e,0xf75a5404,0x254395d8,0x57a8ca0b,0x85b10bd7, + 0x383636ff,0xea2ff723,0x98c4a8f0,0x4add692c,0x7d121756,0xaf0bd68a,0xdde08959,0x0ff94885, + 0xc3cab4d4,0x11d37508,0x63382adb,0xb121eb07,0x86ee957d,0x54f754a1,0x261c0b72,0xf405caae, + 0x4982f786,0x9b9b365a,0xe9706989,0x3b69a855,0x0ca6d62f,0xdebf17f3,0xac544820,0x7e4d89fc, + 0xd39b2fc7,0x0182ee1b,0x7369b1c8,0xa1707014,0x96bf0e6e,0x44a6cfb2,0x364d9061,0xe45451bd, + 0x59d36c95,0x8bcaad49,0xf921f29a,0x2b383346,0x1cf74d3c,0xceee8ce0,0xbc05d333,0x6e1c12ef, + 0xe36982f2,0x3170432e,0x439b1cfd,0x9182dd21,0xa64da35b,0x74546287,0x06bf3d54,0xd4a6fc88, + 0x6921c1a0,0xbb38007c,0xc9d35faf,0x1bca9e73,0x2c05e009,0xfe1c21d5,0x8cf77e06,0x5eeebfda, + 0xf33819e1,0x2121d83d,0x53ca87ee,0x81d34632,0xb61c3848,0x6405f994,0x16eea647,0xc4f7679b, + 0x79705ab3,0xab699b6f,0xd982c4bc,0x0b9b0560,0x3c547b1a,0xee4dbac6,0x9ca6e515,0x4ebf24c9}, + +{0x00000000,0x01d8ac87,0x03b1590e,0x0269f589,0x0762b21c,0x06ba1e9b,0x04d3eb12,0x050b4795, + 0x0ec56438,0x0f1dc8bf,0x0d743d36,0x0cac91b1,0x09a7d624,0x087f7aa3,0x0a168f2a,0x0bce23ad, + 0x1d8ac870,0x1c5264f7,0x1e3b917e,0x1fe33df9,0x1ae87a6c,0x1b30d6eb,0x19592362,0x18818fe5, + 0x134fac48,0x129700cf,0x10fef546,0x112659c1,0x142d1e54,0x15f5b2d3,0x179c475a,0x1644ebdd, + 0x3b1590e0,0x3acd3c67,0x38a4c9ee,0x397c6569,0x3c7722fc,0x3daf8e7b,0x3fc67bf2,0x3e1ed775, + 0x35d0f4d8,0x3408585f,0x3661add6,0x37b90151,0x32b246c4,0x336aea43,0x31031fca,0x30dbb34d, + 0x269f5890,0x2747f417,0x252e019e,0x24f6ad19,0x21fdea8c,0x2025460b,0x224cb382,0x23941f05, + 0x285a3ca8,0x2982902f,0x2beb65a6,0x2a33c921,0x2f388eb4,0x2ee02233,0x2c89d7ba,0x2d517b3d, + 0x762b21c0,0x77f38d47,0x759a78ce,0x7442d449,0x714993dc,0x70913f5b,0x72f8cad2,0x73206655, + 0x78ee45f8,0x7936e97f,0x7b5f1cf6,0x7a87b071,0x7f8cf7e4,0x7e545b63,0x7c3daeea,0x7de5026d, + 0x6ba1e9b0,0x6a794537,0x6810b0be,0x69c81c39,0x6cc35bac,0x6d1bf72b,0x6f7202a2,0x6eaaae25, + 0x65648d88,0x64bc210f,0x66d5d486,0x670d7801,0x62063f94,0x63de9313,0x61b7669a,0x606fca1d, + 0x4d3eb120,0x4ce61da7,0x4e8fe82e,0x4f5744a9,0x4a5c033c,0x4b84afbb,0x49ed5a32,0x4835f6b5, + 0x43fbd518,0x4223799f,0x404a8c16,0x41922091,0x44996704,0x4541cb83,0x47283e0a,0x46f0928d, + 0x50b47950,0x516cd5d7,0x5305205e,0x52dd8cd9,0x57d6cb4c,0x560e67cb,0x54679242,0x55bf3ec5, + 0x5e711d68,0x5fa9b1ef,0x5dc04466,0x5c18e8e1,0x5913af74,0x58cb03f3,0x5aa2f67a,0x5b7a5afd, + 0xec564380,0xed8eef07,0xefe71a8e,0xee3fb609,0xeb34f19c,0xeaec5d1b,0xe885a892,0xe95d0415, + 0xe29327b8,0xe34b8b3f,0xe1227eb6,0xe0fad231,0xe5f195a4,0xe4293923,0xe640ccaa,0xe798602d, + 0xf1dc8bf0,0xf0042777,0xf26dd2fe,0xf3b57e79,0xf6be39ec,0xf766956b,0xf50f60e2,0xf4d7cc65, + 0xff19efc8,0xfec1434f,0xfca8b6c6,0xfd701a41,0xf87b5dd4,0xf9a3f153,0xfbca04da,0xfa12a85d, + 0xd743d360,0xd69b7fe7,0xd4f28a6e,0xd52a26e9,0xd021617c,0xd1f9cdfb,0xd3903872,0xd24894f5, + 0xd986b758,0xd85e1bdf,0xda37ee56,0xdbef42d1,0xdee40544,0xdf3ca9c3,0xdd555c4a,0xdc8df0cd, + 0xcac91b10,0xcb11b797,0xc978421e,0xc8a0ee99,0xcdaba90c,0xcc73058b,0xce1af002,0xcfc25c85, + 0xc40c7f28,0xc5d4d3af,0xc7bd2626,0xc6658aa1,0xc36ecd34,0xc2b661b3,0xc0df943a,0xc10738bd, + 0x9a7d6240,0x9ba5cec7,0x99cc3b4e,0x981497c9,0x9d1fd05c,0x9cc77cdb,0x9eae8952,0x9f7625d5, + 0x94b80678,0x9560aaff,0x97095f76,0x96d1f3f1,0x93dab464,0x920218e3,0x906bed6a,0x91b341ed, + 0x87f7aa30,0x862f06b7,0x8446f33e,0x859e5fb9,0x8095182c,0x814db4ab,0x83244122,0x82fceda5, + 0x8932ce08,0x88ea628f,0x8a839706,0x8b5b3b81,0x8e507c14,0x8f88d093,0x8de1251a,0x8c39899d, + 0xa168f2a0,0xa0b05e27,0xa2d9abae,0xa3010729,0xa60a40bc,0xa7d2ec3b,0xa5bb19b2,0xa463b535, + 0xafad9698,0xae753a1f,0xac1ccf96,0xadc46311,0xa8cf2484,0xa9178803,0xab7e7d8a,0xaaa6d10d, + 0xbce23ad0,0xbd3a9657,0xbf5363de,0xbe8bcf59,0xbb8088cc,0xba58244b,0xb831d1c2,0xb9e97d45, + 0xb2275ee8,0xb3fff26f,0xb19607e6,0xb04eab61,0xb545ecf4,0xb49d4073,0xb6f4b5fa,0xb72c197d}, + +{0x00000000,0xdc6d9ab7,0xbc1a28d9,0x6077b26e,0x7cf54c05,0xa098d6b2,0xc0ef64dc,0x1c82fe6b, + 0xf9ea980a,0x258702bd,0x45f0b0d3,0x999d2a64,0x851fd40f,0x59724eb8,0x3905fcd6,0xe5686661, + 0xf7142da3,0x2b79b714,0x4b0e057a,0x97639fcd,0x8be161a6,0x578cfb11,0x37fb497f,0xeb96d3c8, + 0x0efeb5a9,0xd2932f1e,0xb2e49d70,0x6e8907c7,0x720bf9ac,0xae66631b,0xce11d175,0x127c4bc2, + 0xeae946f1,0x3684dc46,0x56f36e28,0x8a9ef49f,0x961c0af4,0x4a719043,0x2a06222d,0xf66bb89a, + 0x1303defb,0xcf6e444c,0xaf19f622,0x73746c95,0x6ff692fe,0xb39b0849,0xd3ecba27,0x0f812090, + 0x1dfd6b52,0xc190f1e5,0xa1e7438b,0x7d8ad93c,0x61082757,0xbd65bde0,0xdd120f8e,0x017f9539, + 0xe417f358,0x387a69ef,0x580ddb81,0x84604136,0x98e2bf5d,0x448f25ea,0x24f89784,0xf8950d33, + 0xd1139055,0x0d7e0ae2,0x6d09b88c,0xb164223b,0xade6dc50,0x718b46e7,0x11fcf489,0xcd916e3e, + 0x28f9085f,0xf49492e8,0x94e32086,0x488eba31,0x540c445a,0x8861deed,0xe8166c83,0x347bf634, + 0x2607bdf6,0xfa6a2741,0x9a1d952f,0x46700f98,0x5af2f1f3,0x869f6b44,0xe6e8d92a,0x3a85439d, + 0xdfed25fc,0x0380bf4b,0x63f70d25,0xbf9a9792,0xa31869f9,0x7f75f34e,0x1f024120,0xc36fdb97, + 0x3bfad6a4,0xe7974c13,0x87e0fe7d,0x5b8d64ca,0x470f9aa1,0x9b620016,0xfb15b278,0x277828cf, + 0xc2104eae,0x1e7dd419,0x7e0a6677,0xa267fcc0,0xbee502ab,0x6288981c,0x02ff2a72,0xde92b0c5, + 0xcceefb07,0x108361b0,0x70f4d3de,0xac994969,0xb01bb702,0x6c762db5,0x0c019fdb,0xd06c056c, + 0x3504630d,0xe969f9ba,0x891e4bd4,0x5573d163,0x49f12f08,0x959cb5bf,0xf5eb07d1,0x29869d66, + 0xa6e63d1d,0x7a8ba7aa,0x1afc15c4,0xc6918f73,0xda137118,0x067eebaf,0x660959c1,0xba64c376, + 0x5f0ca517,0x83613fa0,0xe3168dce,0x3f7b1779,0x23f9e912,0xff9473a5,0x9fe3c1cb,0x438e5b7c, + 0x51f210be,0x8d9f8a09,0xede83867,0x3185a2d0,0x2d075cbb,0xf16ac60c,0x911d7462,0x4d70eed5, + 0xa81888b4,0x74751203,0x1402a06d,0xc86f3ada,0xd4edc4b1,0x08805e06,0x68f7ec68,0xb49a76df, + 0x4c0f7bec,0x9062e15b,0xf0155335,0x2c78c982,0x30fa37e9,0xec97ad5e,0x8ce01f30,0x508d8587, + 0xb5e5e3e6,0x69887951,0x09ffcb3f,0xd5925188,0xc910afe3,0x157d3554,0x750a873a,0xa9671d8d, + 0xbb1b564f,0x6776ccf8,0x07017e96,0xdb6ce421,0xc7ee1a4a,0x1b8380fd,0x7bf43293,0xa799a824, + 0x42f1ce45,0x9e9c54f2,0xfeebe69c,0x22867c2b,0x3e048240,0xe26918f7,0x821eaa99,0x5e73302e, + 0x77f5ad48,0xab9837ff,0xcbef8591,0x17821f26,0x0b00e14d,0xd76d7bfa,0xb71ac994,0x6b775323, + 0x8e1f3542,0x5272aff5,0x32051d9b,0xee68872c,0xf2ea7947,0x2e87e3f0,0x4ef0519e,0x929dcb29, + 0x80e180eb,0x5c8c1a5c,0x3cfba832,0xe0963285,0xfc14ccee,0x20795659,0x400ee437,0x9c637e80, + 0x790b18e1,0xa5668256,0xc5113038,0x197caa8f,0x05fe54e4,0xd993ce53,0xb9e47c3d,0x6589e68a, + 0x9d1cebb9,0x4171710e,0x2106c360,0xfd6b59d7,0xe1e9a7bc,0x3d843d0b,0x5df38f65,0x819e15d2, + 0x64f673b3,0xb89be904,0xd8ec5b6a,0x0481c1dd,0x18033fb6,0xc46ea501,0xa419176f,0x78748dd8, + 0x6a08c61a,0xb6655cad,0xd612eec3,0x0a7f7474,0x16fd8a1f,0xca9010a8,0xaae7a2c6,0x768a3871, + 0x93e25e10,0x4f8fc4a7,0x2ff876c9,0xf395ec7e,0xef171215,0x337a88a2,0x530d3acc,0x8f60a07b}, + +{0x00000000,0x490d678d,0x921acf1a,0xdb17a897,0x20f48383,0x69f9e40e,0xb2ee4c99,0xfbe32b14, + 0x41e90706,0x08e4608b,0xd3f3c81c,0x9afeaf91,0x611d8485,0x2810e308,0xf3074b9f,0xba0a2c12, + 0x83d20e0c,0xcadf6981,0x11c8c116,0x58c5a69b,0xa3268d8f,0xea2bea02,0x313c4295,0x78312518, + 0xc23b090a,0x8b366e87,0x5021c610,0x192ca19d,0xe2cf8a89,0xabc2ed04,0x70d54593,0x39d8221e, + 0x036501af,0x4a686622,0x917fceb5,0xd872a938,0x2391822c,0x6a9ce5a1,0xb18b4d36,0xf8862abb, + 0x428c06a9,0x0b816124,0xd096c9b3,0x999bae3e,0x6278852a,0x2b75e2a7,0xf0624a30,0xb96f2dbd, + 0x80b70fa3,0xc9ba682e,0x12adc0b9,0x5ba0a734,0xa0438c20,0xe94eebad,0x3259433a,0x7b5424b7, + 0xc15e08a5,0x88536f28,0x5344c7bf,0x1a49a032,0xe1aa8b26,0xa8a7ecab,0x73b0443c,0x3abd23b1, + 0x06ca035e,0x4fc764d3,0x94d0cc44,0xddddabc9,0x263e80dd,0x6f33e750,0xb4244fc7,0xfd29284a, + 0x47230458,0x0e2e63d5,0xd539cb42,0x9c34accf,0x67d787db,0x2edae056,0xf5cd48c1,0xbcc02f4c, + 0x85180d52,0xcc156adf,0x1702c248,0x5e0fa5c5,0xa5ec8ed1,0xece1e95c,0x37f641cb,0x7efb2646, + 0xc4f10a54,0x8dfc6dd9,0x56ebc54e,0x1fe6a2c3,0xe40589d7,0xad08ee5a,0x761f46cd,0x3f122140, + 0x05af02f1,0x4ca2657c,0x97b5cdeb,0xdeb8aa66,0x255b8172,0x6c56e6ff,0xb7414e68,0xfe4c29e5, + 0x444605f7,0x0d4b627a,0xd65ccaed,0x9f51ad60,0x64b28674,0x2dbfe1f9,0xf6a8496e,0xbfa52ee3, + 0x867d0cfd,0xcf706b70,0x1467c3e7,0x5d6aa46a,0xa6898f7e,0xef84e8f3,0x34934064,0x7d9e27e9, + 0xc7940bfb,0x8e996c76,0x558ec4e1,0x1c83a36c,0xe7608878,0xae6deff5,0x757a4762,0x3c7720ef, + 0x0d9406bc,0x44996131,0x9f8ec9a6,0xd683ae2b,0x2d60853f,0x646de2b2,0xbf7a4a25,0xf6772da8, + 0x4c7d01ba,0x05706637,0xde67cea0,0x976aa92d,0x6c898239,0x2584e5b4,0xfe934d23,0xb79e2aae, + 0x8e4608b0,0xc74b6f3d,0x1c5cc7aa,0x5551a027,0xaeb28b33,0xe7bfecbe,0x3ca84429,0x75a523a4, + 0xcfaf0fb6,0x86a2683b,0x5db5c0ac,0x14b8a721,0xef5b8c35,0xa656ebb8,0x7d41432f,0x344c24a2, + 0x0ef10713,0x47fc609e,0x9cebc809,0xd5e6af84,0x2e058490,0x6708e31d,0xbc1f4b8a,0xf5122c07, + 0x4f180015,0x06156798,0xdd02cf0f,0x940fa882,0x6fec8396,0x26e1e41b,0xfdf64c8c,0xb4fb2b01, + 0x8d23091f,0xc42e6e92,0x1f39c605,0x5634a188,0xadd78a9c,0xe4daed11,0x3fcd4586,0x76c0220b, + 0xccca0e19,0x85c76994,0x5ed0c103,0x17dda68e,0xec3e8d9a,0xa533ea17,0x7e244280,0x3729250d, + 0x0b5e05e2,0x4253626f,0x9944caf8,0xd049ad75,0x2baa8661,0x62a7e1ec,0xb9b0497b,0xf0bd2ef6, + 0x4ab702e4,0x03ba6569,0xd8adcdfe,0x91a0aa73,0x6a438167,0x234ee6ea,0xf8594e7d,0xb15429f0, + 0x888c0bee,0xc1816c63,0x1a96c4f4,0x539ba379,0xa878886d,0xe175efe0,0x3a624777,0x736f20fa, + 0xc9650ce8,0x80686b65,0x5b7fc3f2,0x1272a47f,0xe9918f6b,0xa09ce8e6,0x7b8b4071,0x328627fc, + 0x083b044d,0x413663c0,0x9a21cb57,0xd32cacda,0x28cf87ce,0x61c2e043,0xbad548d4,0xf3d82f59, + 0x49d2034b,0x00df64c6,0xdbc8cc51,0x92c5abdc,0x692680c8,0x202be745,0xfb3c4fd2,0xb231285f, + 0x8be90a41,0xc2e46dcc,0x19f3c55b,0x50fea2d6,0xab1d89c2,0xe210ee4f,0x390746d8,0x700a2155, + 0xca000d47,0x830d6aca,0x581ac25d,0x1117a5d0,0xeaf48ec4,0xa3f9e949,0x78ee41de,0x31e32653}, + +{0x00000000,0x1b280d78,0x36501af0,0x2d781788,0x6ca035e0,0x77883898,0x5af02f10,0x41d82268, + 0xd9406bc0,0xc26866b8,0xef107130,0xf4387c48,0xb5e05e20,0xaec85358,0x83b044d0,0x989849a8, + 0xb641ca37,0xad69c74f,0x8011d0c7,0x9b39ddbf,0xdae1ffd7,0xc1c9f2af,0xecb1e527,0xf799e85f, + 0x6f01a1f7,0x7429ac8f,0x5951bb07,0x4279b67f,0x03a19417,0x1889996f,0x35f18ee7,0x2ed9839f, + 0x684289d9,0x736a84a1,0x5e129329,0x453a9e51,0x04e2bc39,0x1fcab141,0x32b2a6c9,0x299aabb1, + 0xb102e219,0xaa2aef61,0x8752f8e9,0x9c7af591,0xdda2d7f9,0xc68ada81,0xebf2cd09,0xf0dac071, + 0xde0343ee,0xc52b4e96,0xe853591e,0xf37b5466,0xb2a3760e,0xa98b7b76,0x84f36cfe,0x9fdb6186, + 0x0743282e,0x1c6b2556,0x311332de,0x2a3b3fa6,0x6be31dce,0x70cb10b6,0x5db3073e,0x469b0a46, + 0xd08513b2,0xcbad1eca,0xe6d50942,0xfdfd043a,0xbc252652,0xa70d2b2a,0x8a753ca2,0x915d31da, + 0x09c57872,0x12ed750a,0x3f956282,0x24bd6ffa,0x65654d92,0x7e4d40ea,0x53355762,0x481d5a1a, + 0x66c4d985,0x7decd4fd,0x5094c375,0x4bbcce0d,0x0a64ec65,0x114ce11d,0x3c34f695,0x271cfbed, + 0xbf84b245,0xa4acbf3d,0x89d4a8b5,0x92fca5cd,0xd32487a5,0xc80c8add,0xe5749d55,0xfe5c902d, + 0xb8c79a6b,0xa3ef9713,0x8e97809b,0x95bf8de3,0xd467af8b,0xcf4fa2f3,0xe237b57b,0xf91fb803, + 0x6187f1ab,0x7aaffcd3,0x57d7eb5b,0x4cffe623,0x0d27c44b,0x160fc933,0x3b77debb,0x205fd3c3, + 0x0e86505c,0x15ae5d24,0x38d64aac,0x23fe47d4,0x622665bc,0x790e68c4,0x54767f4c,0x4f5e7234, + 0xd7c63b9c,0xccee36e4,0xe196216c,0xfabe2c14,0xbb660e7c,0xa04e0304,0x8d36148c,0x961e19f4, + 0xa5cb3ad3,0xbee337ab,0x939b2023,0x88b32d5b,0xc96b0f33,0xd243024b,0xff3b15c3,0xe41318bb, + 0x7c8b5113,0x67a35c6b,0x4adb4be3,0x51f3469b,0x102b64f3,0x0b03698b,0x267b7e03,0x3d53737b, + 0x138af0e4,0x08a2fd9c,0x25daea14,0x3ef2e76c,0x7f2ac504,0x6402c87c,0x497adff4,0x5252d28c, + 0xcaca9b24,0xd1e2965c,0xfc9a81d4,0xe7b28cac,0xa66aaec4,0xbd42a3bc,0x903ab434,0x8b12b94c, + 0xcd89b30a,0xd6a1be72,0xfbd9a9fa,0xe0f1a482,0xa12986ea,0xba018b92,0x97799c1a,0x8c519162, + 0x14c9d8ca,0x0fe1d5b2,0x2299c23a,0x39b1cf42,0x7869ed2a,0x6341e052,0x4e39f7da,0x5511faa2, + 0x7bc8793d,0x60e07445,0x4d9863cd,0x56b06eb5,0x17684cdd,0x0c4041a5,0x2138562d,0x3a105b55, + 0xa28812fd,0xb9a01f85,0x94d8080d,0x8ff00575,0xce28271d,0xd5002a65,0xf8783ded,0xe3503095, + 0x754e2961,0x6e662419,0x431e3391,0x58363ee9,0x19ee1c81,0x02c611f9,0x2fbe0671,0x34960b09, + 0xac0e42a1,0xb7264fd9,0x9a5e5851,0x81765529,0xc0ae7741,0xdb867a39,0xf6fe6db1,0xedd660c9, + 0xc30fe356,0xd827ee2e,0xf55ff9a6,0xee77f4de,0xafafd6b6,0xb487dbce,0x99ffcc46,0x82d7c13e, + 0x1a4f8896,0x016785ee,0x2c1f9266,0x37379f1e,0x76efbd76,0x6dc7b00e,0x40bfa786,0x5b97aafe, + 0x1d0ca0b8,0x0624adc0,0x2b5cba48,0x3074b730,0x71ac9558,0x6a849820,0x47fc8fa8,0x5cd482d0, + 0xc44ccb78,0xdf64c600,0xf21cd188,0xe934dcf0,0xa8ecfe98,0xb3c4f3e0,0x9ebce468,0x8594e910, + 0xab4d6a8f,0xb06567f7,0x9d1d707f,0x86357d07,0xc7ed5f6f,0xdcc55217,0xf1bd459f,0xea9548e7, + 0x720d014f,0x69250c37,0x445d1bbf,0x5f7516c7,0x1ead34af,0x058539d7,0x28fd2e5f,0x33d52327}, + +{0x00000000,0x4f576811,0x9eaed022,0xd1f9b833,0x399cbdf3,0x76cbd5e2,0xa7326dd1,0xe86505c0, + 0x73397be6,0x3c6e13f7,0xed97abc4,0xa2c0c3d5,0x4aa5c615,0x05f2ae04,0xd40b1637,0x9b5c7e26, + 0xe672f7cc,0xa9259fdd,0x78dc27ee,0x378b4fff,0xdfee4a3f,0x90b9222e,0x41409a1d,0x0e17f20c, + 0x954b8c2a,0xda1ce43b,0x0be55c08,0x44b23419,0xacd731d9,0xe38059c8,0x3279e1fb,0x7d2e89ea, + 0xc824f22f,0x87739a3e,0x568a220d,0x19dd4a1c,0xf1b84fdc,0xbeef27cd,0x6f169ffe,0x2041f7ef, + 0xbb1d89c9,0xf44ae1d8,0x25b359eb,0x6ae431fa,0x8281343a,0xcdd65c2b,0x1c2fe418,0x53788c09, + 0x2e5605e3,0x61016df2,0xb0f8d5c1,0xffafbdd0,0x17cab810,0x589dd001,0x89646832,0xc6330023, + 0x5d6f7e05,0x12381614,0xc3c1ae27,0x8c96c636,0x64f3c3f6,0x2ba4abe7,0xfa5d13d4,0xb50a7bc5, + 0x9488f9e9,0xdbdf91f8,0x0a2629cb,0x457141da,0xad14441a,0xe2432c0b,0x33ba9438,0x7cedfc29, + 0xe7b1820f,0xa8e6ea1e,0x791f522d,0x36483a3c,0xde2d3ffc,0x917a57ed,0x4083efde,0x0fd487cf, + 0x72fa0e25,0x3dad6634,0xec54de07,0xa303b616,0x4b66b3d6,0x0431dbc7,0xd5c863f4,0x9a9f0be5, + 0x01c375c3,0x4e941dd2,0x9f6da5e1,0xd03acdf0,0x385fc830,0x7708a021,0xa6f11812,0xe9a67003, + 0x5cac0bc6,0x13fb63d7,0xc202dbe4,0x8d55b3f5,0x6530b635,0x2a67de24,0xfb9e6617,0xb4c90e06, + 0x2f957020,0x60c21831,0xb13ba002,0xfe6cc813,0x1609cdd3,0x595ea5c2,0x88a71df1,0xc7f075e0, + 0xbadefc0a,0xf589941b,0x24702c28,0x6b274439,0x834241f9,0xcc1529e8,0x1dec91db,0x52bbf9ca, + 0xc9e787ec,0x86b0effd,0x574957ce,0x181e3fdf,0xf07b3a1f,0xbf2c520e,0x6ed5ea3d,0x2182822c, + 0x2dd0ee65,0x62878674,0xb37e3e47,0xfc295656,0x144c5396,0x5b1b3b87,0x8ae283b4,0xc5b5eba5, + 0x5ee99583,0x11befd92,0xc04745a1,0x8f102db0,0x67752870,0x28224061,0xf9dbf852,0xb68c9043, + 0xcba219a9,0x84f571b8,0x550cc98b,0x1a5ba19a,0xf23ea45a,0xbd69cc4b,0x6c907478,0x23c71c69, + 0xb89b624f,0xf7cc0a5e,0x2635b26d,0x6962da7c,0x8107dfbc,0xce50b7ad,0x1fa90f9e,0x50fe678f, + 0xe5f41c4a,0xaaa3745b,0x7b5acc68,0x340da479,0xdc68a1b9,0x933fc9a8,0x42c6719b,0x0d91198a, + 0x96cd67ac,0xd99a0fbd,0x0863b78e,0x4734df9f,0xaf51da5f,0xe006b24e,0x31ff0a7d,0x7ea8626c, + 0x0386eb86,0x4cd18397,0x9d283ba4,0xd27f53b5,0x3a1a5675,0x754d3e64,0xa4b48657,0xebe3ee46, + 0x70bf9060,0x3fe8f871,0xee114042,0xa1462853,0x49232d93,0x06744582,0xd78dfdb1,0x98da95a0, + 0xb958178c,0xf60f7f9d,0x27f6c7ae,0x68a1afbf,0x80c4aa7f,0xcf93c26e,0x1e6a7a5d,0x513d124c, + 0xca616c6a,0x8536047b,0x54cfbc48,0x1b98d459,0xf3fdd199,0xbcaab988,0x6d5301bb,0x220469aa, + 0x5f2ae040,0x107d8851,0xc1843062,0x8ed35873,0x66b65db3,0x29e135a2,0xf8188d91,0xb74fe580, + 0x2c139ba6,0x6344f3b7,0xb2bd4b84,0xfdea2395,0x158f2655,0x5ad84e44,0x8b21f677,0xc4769e66, + 0x717ce5a3,0x3e2b8db2,0xefd23581,0xa0855d90,0x48e05850,0x07b73041,0xd64e8872,0x9919e063, + 0x02459e45,0x4d12f654,0x9ceb4e67,0xd3bc2676,0x3bd923b6,0x748e4ba7,0xa577f394,0xea209b85, + 0x970e126f,0xd8597a7e,0x09a0c24d,0x46f7aa5c,0xae92af9c,0xe1c5c78d,0x303c7fbe,0x7f6b17af, + 0xe4376989,0xab600198,0x7a99b9ab,0x35ced1ba,0xddabd47a,0x92fcbc6b,0x43050458,0x0c526c49}, + +{0x00000000,0x5ba1dcca,0xb743b994,0xece2655e,0x6a466e9f,0x31e7b255,0xdd05d70b,0x86a40bc1, + 0xd48cdd3e,0x8f2d01f4,0x63cf64aa,0x386eb860,0xbecab3a1,0xe56b6f6b,0x09890a35,0x5228d6ff, + 0xadd8a7cb,0xf6797b01,0x1a9b1e5f,0x413ac295,0xc79ec954,0x9c3f159e,0x70dd70c0,0x2b7cac0a, + 0x79547af5,0x22f5a63f,0xce17c361,0x95b61fab,0x1312146a,0x48b3c8a0,0xa451adfe,0xfff07134, + 0x5f705221,0x04d18eeb,0xe833ebb5,0xb392377f,0x35363cbe,0x6e97e074,0x8275852a,0xd9d459e0, + 0x8bfc8f1f,0xd05d53d5,0x3cbf368b,0x671eea41,0xe1bae180,0xba1b3d4a,0x56f95814,0x0d5884de, + 0xf2a8f5ea,0xa9092920,0x45eb4c7e,0x1e4a90b4,0x98ee9b75,0xc34f47bf,0x2fad22e1,0x740cfe2b, + 0x262428d4,0x7d85f41e,0x91679140,0xcac64d8a,0x4c62464b,0x17c39a81,0xfb21ffdf,0xa0802315, + 0xbee0a442,0xe5417888,0x09a31dd6,0x5202c11c,0xd4a6cadd,0x8f071617,0x63e57349,0x3844af83, + 0x6a6c797c,0x31cda5b6,0xdd2fc0e8,0x868e1c22,0x002a17e3,0x5b8bcb29,0xb769ae77,0xecc872bd, + 0x13380389,0x4899df43,0xa47bba1d,0xffda66d7,0x797e6d16,0x22dfb1dc,0xce3dd482,0x959c0848, + 0xc7b4deb7,0x9c15027d,0x70f76723,0x2b56bbe9,0xadf2b028,0xf6536ce2,0x1ab109bc,0x4110d576, + 0xe190f663,0xba312aa9,0x56d34ff7,0x0d72933d,0x8bd698fc,0xd0774436,0x3c952168,0x6734fda2, + 0x351c2b5d,0x6ebdf797,0x825f92c9,0xd9fe4e03,0x5f5a45c2,0x04fb9908,0xe819fc56,0xb3b8209c, + 0x4c4851a8,0x17e98d62,0xfb0be83c,0xa0aa34f6,0x260e3f37,0x7dafe3fd,0x914d86a3,0xcaec5a69, + 0x98c48c96,0xc365505c,0x2f873502,0x7426e9c8,0xf282e209,0xa9233ec3,0x45c15b9d,0x1e608757, + 0x79005533,0x22a189f9,0xce43eca7,0x95e2306d,0x13463bac,0x48e7e766,0xa4058238,0xffa45ef2, + 0xad8c880d,0xf62d54c7,0x1acf3199,0x416eed53,0xc7cae692,0x9c6b3a58,0x70895f06,0x2b2883cc, + 0xd4d8f2f8,0x8f792e32,0x639b4b6c,0x383a97a6,0xbe9e9c67,0xe53f40ad,0x09dd25f3,0x527cf939, + 0x00542fc6,0x5bf5f30c,0xb7179652,0xecb64a98,0x6a124159,0x31b39d93,0xdd51f8cd,0x86f02407, + 0x26700712,0x7dd1dbd8,0x9133be86,0xca92624c,0x4c36698d,0x1797b547,0xfb75d019,0xa0d40cd3, + 0xf2fcda2c,0xa95d06e6,0x45bf63b8,0x1e1ebf72,0x98bab4b3,0xc31b6879,0x2ff90d27,0x7458d1ed, + 0x8ba8a0d9,0xd0097c13,0x3ceb194d,0x674ac587,0xe1eece46,0xba4f128c,0x56ad77d2,0x0d0cab18, + 0x5f247de7,0x0485a12d,0xe867c473,0xb3c618b9,0x35621378,0x6ec3cfb2,0x8221aaec,0xd9807626, + 0xc7e0f171,0x9c412dbb,0x70a348e5,0x2b02942f,0xada69fee,0xf6074324,0x1ae5267a,0x4144fab0, + 0x136c2c4f,0x48cdf085,0xa42f95db,0xff8e4911,0x792a42d0,0x228b9e1a,0xce69fb44,0x95c8278e, + 0x6a3856ba,0x31998a70,0xdd7bef2e,0x86da33e4,0x007e3825,0x5bdfe4ef,0xb73d81b1,0xec9c5d7b, + 0xbeb48b84,0xe515574e,0x09f73210,0x5256eeda,0xd4f2e51b,0x8f5339d1,0x63b15c8f,0x38108045, + 0x9890a350,0xc3317f9a,0x2fd31ac4,0x7472c60e,0xf2d6cdcf,0xa9771105,0x4595745b,0x1e34a891, + 0x4c1c7e6e,0x17bda2a4,0xfb5fc7fa,0xa0fe1b30,0x265a10f1,0x7dfbcc3b,0x9119a965,0xcab875af, + 0x3548049b,0x6ee9d851,0x820bbd0f,0xd9aa61c5,0x5f0e6a04,0x04afb6ce,0xe84dd390,0xb3ec0f5a, + 0xe1c4d9a5,0xba65056f,0x56876031,0x0d26bcfb,0x8b82b73a,0xd0236bf0,0x3cc10eae,0x6760d264}}; diff --git a/thirdparty/libogg/framing.c b/thirdparty/libogg/framing.c index 79fc715c8c6..83601199ad8 100644 --- a/thirdparty/libogg/framing.c +++ b/thirdparty/libogg/framing.c @@ -5,14 +5,13 @@ * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * * * - * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2010 * + * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2018 * * by the Xiph.Org Foundation http://www.xiph.org/ * * * ******************************************************************** function: code raw packets into framed OggSquish stream and decode Ogg streams back into raw packets - last mod: $Id$ note: The CRC code is directly derived from public domain code by Ross Williams (ross@guest.adelaide.edu.au). See docs/framing.html @@ -20,6 +19,10 @@ ********************************************************************/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + #include #include #include @@ -45,7 +48,7 @@ int ogg_page_eos(const ogg_page *og){ ogg_int64_t ogg_page_granulepos(const ogg_page *og){ unsigned char *page=og->header; - ogg_int64_t granulepos=page[13]&(0xff); + ogg_uint64_t granulepos=page[13]&(0xff); granulepos= (granulepos<<8)|(page[12]&0xff); granulepos= (granulepos<<8)|(page[11]&0xff); granulepos= (granulepos<<8)|(page[10]&0xff); @@ -53,21 +56,21 @@ ogg_int64_t ogg_page_granulepos(const ogg_page *og){ granulepos= (granulepos<<8)|(page[8]&0xff); granulepos= (granulepos<<8)|(page[7]&0xff); granulepos= (granulepos<<8)|(page[6]&0xff); - return(granulepos); + return((ogg_int64_t)granulepos); } int ogg_page_serialno(const ogg_page *og){ - return(og->header[14] | - (og->header[15]<<8) | - (og->header[16]<<16) | - (og->header[17]<<24)); + return((int)((ogg_uint32_t)og->header[14]) | + ((ogg_uint32_t)og->header[15]<<8) | + ((ogg_uint32_t)og->header[16]<<16) | + ((ogg_uint32_t)og->header[17]<<24)); } long ogg_page_pageno(const ogg_page *og){ - return(og->header[18] | - (og->header[19]<<8) | - (og->header[20]<<16) | - (og->header[21]<<24)); + return((long)((ogg_uint32_t)og->header[18]) | + ((ogg_uint32_t)og->header[19]<<8) | + ((ogg_uint32_t)og->header[20]<<16) | + ((ogg_uint32_t)og->header[21]<<24)); } @@ -99,90 +102,31 @@ int ogg_page_packets(const ogg_page *og){ #if 0 /* helper to initialize lookup for direct-table CRC (illustrative; we - use the static init below) */ + use the static init in crctable.h) */ -static ogg_uint32_t _ogg_crc_entry(unsigned long index){ - int i; - unsigned long r; +static void _ogg_crc_init(){ + int i, j; + ogg_uint32_t polynomial, crc; + polynomial = 0x04c11db7; /* The same as the ethernet generator + polynomial, although we use an + unreflected alg and an init/final + of 0, not 0xffffffff */ + for (i = 0; i <= 0xFF; i++){ + crc = i << 24; - r = index << 24; - for (i=0; i<8; i++) - if (r & 0x80000000UL) - r = (r << 1) ^ 0x04c11db7; /* The same as the ethernet generator - polynomial, although we use an - unreflected alg and an init/final - of 0, not 0xffffffff */ - else - r<<=1; - return (r & 0xffffffffUL); + for (j = 0; j < 8; j++) + crc = (crc << 1) ^ (crc & (1 << 31) ? polynomial : 0); + + crc_lookup[0][i] = crc; + } + + for (i = 0; i <= 0xFF; i++) + for (j = 1; j < 8; j++) + crc_lookup[j][i] = crc_lookup[0][(crc_lookup[j - 1][i] >> 24) & 0xFF] ^ (crc_lookup[j - 1][i] << 8); } #endif -static const ogg_uint32_t crc_lookup[256]={ - 0x00000000,0x04c11db7,0x09823b6e,0x0d4326d9, - 0x130476dc,0x17c56b6b,0x1a864db2,0x1e475005, - 0x2608edb8,0x22c9f00f,0x2f8ad6d6,0x2b4bcb61, - 0x350c9b64,0x31cd86d3,0x3c8ea00a,0x384fbdbd, - 0x4c11db70,0x48d0c6c7,0x4593e01e,0x4152fda9, - 0x5f15adac,0x5bd4b01b,0x569796c2,0x52568b75, - 0x6a1936c8,0x6ed82b7f,0x639b0da6,0x675a1011, - 0x791d4014,0x7ddc5da3,0x709f7b7a,0x745e66cd, - 0x9823b6e0,0x9ce2ab57,0x91a18d8e,0x95609039, - 0x8b27c03c,0x8fe6dd8b,0x82a5fb52,0x8664e6e5, - 0xbe2b5b58,0xbaea46ef,0xb7a96036,0xb3687d81, - 0xad2f2d84,0xa9ee3033,0xa4ad16ea,0xa06c0b5d, - 0xd4326d90,0xd0f37027,0xddb056fe,0xd9714b49, - 0xc7361b4c,0xc3f706fb,0xceb42022,0xca753d95, - 0xf23a8028,0xf6fb9d9f,0xfbb8bb46,0xff79a6f1, - 0xe13ef6f4,0xe5ffeb43,0xe8bccd9a,0xec7dd02d, - 0x34867077,0x30476dc0,0x3d044b19,0x39c556ae, - 0x278206ab,0x23431b1c,0x2e003dc5,0x2ac12072, - 0x128e9dcf,0x164f8078,0x1b0ca6a1,0x1fcdbb16, - 0x018aeb13,0x054bf6a4,0x0808d07d,0x0cc9cdca, - 0x7897ab07,0x7c56b6b0,0x71159069,0x75d48dde, - 0x6b93dddb,0x6f52c06c,0x6211e6b5,0x66d0fb02, - 0x5e9f46bf,0x5a5e5b08,0x571d7dd1,0x53dc6066, - 0x4d9b3063,0x495a2dd4,0x44190b0d,0x40d816ba, - 0xaca5c697,0xa864db20,0xa527fdf9,0xa1e6e04e, - 0xbfa1b04b,0xbb60adfc,0xb6238b25,0xb2e29692, - 0x8aad2b2f,0x8e6c3698,0x832f1041,0x87ee0df6, - 0x99a95df3,0x9d684044,0x902b669d,0x94ea7b2a, - 0xe0b41de7,0xe4750050,0xe9362689,0xedf73b3e, - 0xf3b06b3b,0xf771768c,0xfa325055,0xfef34de2, - 0xc6bcf05f,0xc27dede8,0xcf3ecb31,0xcbffd686, - 0xd5b88683,0xd1799b34,0xdc3abded,0xd8fba05a, - 0x690ce0ee,0x6dcdfd59,0x608edb80,0x644fc637, - 0x7a089632,0x7ec98b85,0x738aad5c,0x774bb0eb, - 0x4f040d56,0x4bc510e1,0x46863638,0x42472b8f, - 0x5c007b8a,0x58c1663d,0x558240e4,0x51435d53, - 0x251d3b9e,0x21dc2629,0x2c9f00f0,0x285e1d47, - 0x36194d42,0x32d850f5,0x3f9b762c,0x3b5a6b9b, - 0x0315d626,0x07d4cb91,0x0a97ed48,0x0e56f0ff, - 0x1011a0fa,0x14d0bd4d,0x19939b94,0x1d528623, - 0xf12f560e,0xf5ee4bb9,0xf8ad6d60,0xfc6c70d7, - 0xe22b20d2,0xe6ea3d65,0xeba91bbc,0xef68060b, - 0xd727bbb6,0xd3e6a601,0xdea580d8,0xda649d6f, - 0xc423cd6a,0xc0e2d0dd,0xcda1f604,0xc960ebb3, - 0xbd3e8d7e,0xb9ff90c9,0xb4bcb610,0xb07daba7, - 0xae3afba2,0xaafbe615,0xa7b8c0cc,0xa379dd7b, - 0x9b3660c6,0x9ff77d71,0x92b45ba8,0x9675461f, - 0x8832161a,0x8cf30bad,0x81b02d74,0x857130c3, - 0x5d8a9099,0x594b8d2e,0x5408abf7,0x50c9b640, - 0x4e8ee645,0x4a4ffbf2,0x470cdd2b,0x43cdc09c, - 0x7b827d21,0x7f436096,0x7200464f,0x76c15bf8, - 0x68860bfd,0x6c47164a,0x61043093,0x65c52d24, - 0x119b4be9,0x155a565e,0x18197087,0x1cd86d30, - 0x029f3d35,0x065e2082,0x0b1d065b,0x0fdc1bec, - 0x3793a651,0x3352bbe6,0x3e119d3f,0x3ad08088, - 0x2497d08d,0x2056cd3a,0x2d15ebe3,0x29d4f654, - 0xc5a92679,0xc1683bce,0xcc2b1d17,0xc8ea00a0, - 0xd6ad50a5,0xd26c4d12,0xdf2f6bcb,0xdbee767c, - 0xe3a1cbc1,0xe760d676,0xea23f0af,0xeee2ed18, - 0xf0a5bd1d,0xf464a0aa,0xf9278673,0xfde69bc4, - 0x89b8fd09,0x8d79e0be,0x803ac667,0x84fbdbd0, - 0x9abc8bd5,0x9e7d9662,0x933eb0bb,0x97ffad0c, - 0xafb010b1,0xab710d06,0xa6322bdf,0xa2f33668, - 0xbcb4666d,0xb8757bda,0xb5365d03,0xb1f740b4}; +#include "crctable.h" /* init the encode/decode logical stream state */ @@ -290,10 +234,27 @@ static int _os_lacing_expand(ogg_stream_state *os,long needed){ /* Direct table CRC; note that this will be faster in the future if we perform the checksum simultaneously with other copies */ +static ogg_uint32_t _os_update_crc(ogg_uint32_t crc, unsigned char *buffer, int size){ + while (size>=8){ + crc^=((ogg_uint32_t)buffer[0]<<24)|((ogg_uint32_t)buffer[1]<<16)|((ogg_uint32_t)buffer[2]<<8)|((ogg_uint32_t)buffer[3]); + + crc=crc_lookup[7][ crc>>24 ]^crc_lookup[6][(crc>>16)&0xFF]^ + crc_lookup[5][(crc>> 8)&0xFF]^crc_lookup[4][ crc &0xFF]^ + crc_lookup[3][buffer[4] ]^crc_lookup[2][buffer[5] ]^ + crc_lookup[1][buffer[6] ]^crc_lookup[0][buffer[7] ]; + + buffer+=8; + size-=8; + } + + while (size--) + crc=(crc<<8)^crc_lookup[0][((crc >> 24)&0xff)^*buffer++]; + return crc; +} + void ogg_page_checksum_set(ogg_page *og){ if(og){ ogg_uint32_t crc_reg=0; - int i; /* safety; needed for API behavior, but not framing code */ og->header[22]=0; @@ -301,10 +262,8 @@ void ogg_page_checksum_set(ogg_page *og){ og->header[24]=0; og->header[25]=0; - for(i=0;iheader_len;i++) - crc_reg=(crc_reg<<8)^crc_lookup[((crc_reg >> 24)&0xff)^og->header[i]]; - for(i=0;ibody_len;i++) - crc_reg=(crc_reg<<8)^crc_lookup[((crc_reg >> 24)&0xff)^og->body[i]]; + crc_reg=_os_update_crc(crc_reg,og->header,og->header_len); + crc_reg=_os_update_crc(crc_reg,og->body,og->body_len); og->header[22]=(unsigned char)(crc_reg&0xff); og->header[23]=(unsigned char)((crc_reg>>8)&0xff); @@ -414,9 +373,9 @@ static int ogg_stream_flush_i(ogg_stream_state *os,ogg_page *og, int force, int }else{ /* The extra packets_done, packet_just_done logic here attempts to do two things: - 1) Don't unneccessarily span pages. + 1) Don't unnecessarily span pages. 2) Unless necessary, don't flush pages if there are less than four packets on - them; this expands page size to reduce unneccessary overhead if incoming packets + them; this expands page size to reduce unnecessary overhead if incoming packets are large. These are not necessary behaviors, just 'always better than naive flushing' without requiring an application to explicitly request a specific optimized @@ -723,16 +682,15 @@ long ogg_sync_pageseek(ogg_sync_state *oy,ogg_page *og){ /* replace the computed checksum with the one actually read in */ memcpy(page+22,chksum,4); +#ifndef DISABLE_CRC /* Bad checksum. Lose sync */ goto sync_fail; +#endif } } /* yes, have a whole page all ready to go */ { - unsigned char *page=oy->data+oy->returned; - long bytes; - if(og){ og->header=page; og->header_len=oy->headerbytes; @@ -1814,6 +1772,7 @@ int main(void){ test_pack(packets,headret,0,0,0); } +#ifndef DISABLE_CRC { /* test for the libogg 1.1.1 resync in large continuation bug found by Josh Coalson) */ @@ -1823,6 +1782,9 @@ int main(void){ fprintf(stderr,"testing continuation resync in very large packets... "); test_pack(packets,headret,100,2,3); } +#else + fprintf(stderr,"Skipping continuation resync test due to --disable-crc\n"); +#endif { /* term only page. why not? */ @@ -2084,6 +2046,7 @@ int main(void){ fprintf(stderr,"ok.\n"); } +#ifndef DISABLE_CRC /* Test recapture: page + garbage + page */ { ogg_page og_de; @@ -2125,6 +2088,9 @@ int main(void){ fprintf(stderr,"ok.\n"); } +#else + fprintf(stderr,"Skipping recapture test due to --disable-crc\n"); +#endif /* Free page data that was previously copied */ { @@ -2133,6 +2099,9 @@ int main(void){ } } } + ogg_sync_clear(&oy); + ogg_stream_clear(&os_en); + ogg_stream_clear(&os_de); return(0); } diff --git a/thirdparty/libogg/ogg/config_types.h b/thirdparty/libogg/ogg/config_types.h index e6306575471..3574a8ad448 100644 --- a/thirdparty/libogg/ogg/config_types.h +++ b/thirdparty/libogg/ogg/config_types.h @@ -8,5 +8,6 @@ typedef uint16_t ogg_uint16_t; typedef int32_t ogg_int32_t; typedef uint32_t ogg_uint32_t; typedef int64_t ogg_int64_t; +typedef uint64_t ogg_uint64_t; #endif diff --git a/thirdparty/libogg/ogg/ogg.h b/thirdparty/libogg/ogg/ogg.h index 7609fc24d67..c4325aa76da 100644 --- a/thirdparty/libogg/ogg/ogg.h +++ b/thirdparty/libogg/ogg/ogg.h @@ -11,7 +11,6 @@ ******************************************************************** function: toplevel libogg include - last mod: $Id$ ********************************************************************/ #ifndef _OGG_H diff --git a/thirdparty/libogg/ogg/os_types.h b/thirdparty/libogg/ogg/os_types.h index b8f56308b50..e655a1d6280 100644 --- a/thirdparty/libogg/ogg/os_types.h +++ b/thirdparty/libogg/ogg/os_types.h @@ -10,8 +10,7 @@ * * ******************************************************************** - function: #ifdef jail to whip a few platforms into the UNIX ideal. - last mod: $Id$ + function: Define a consistent set of types on each platform. ********************************************************************/ #ifndef _OS_TYPES_H @@ -44,6 +43,7 @@ typedef unsigned long long ogg_uint64_t; # elif defined(__MWERKS__) typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef short ogg_int16_t; @@ -62,6 +62,7 @@ typedef __int64 ogg_int64_t; typedef __int32 ogg_int32_t; typedef unsigned __int32 ogg_uint32_t; + typedef unsigned __int64 ogg_uint64_t; typedef __int16 ogg_int16_t; typedef unsigned __int16 ogg_uint16_t; # endif @@ -69,12 +70,13 @@ #elif (defined(__APPLE__) && defined(__MACH__)) /* MacOS X Framework build */ -# include +# include typedef int16_t ogg_int16_t; - typedef uint16_t ogg_uint16_t; + typedef u_int16_t ogg_uint16_t; typedef int32_t ogg_int32_t; - typedef uint32_t ogg_uint32_t; + typedef u_int32_t ogg_uint32_t; typedef int64_t ogg_int64_t; + typedef u_int64_t ogg_uint64_t; #elif defined(__HAIKU__) @@ -85,6 +87,7 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; #elif defined(__BEOS__) @@ -95,6 +98,7 @@ typedef int32_t ogg_int32_t; typedef uint32_t ogg_uint32_t; typedef int64_t ogg_int64_t; + typedef uint64_t ogg_uint64_t; #elif defined (__EMX__) @@ -104,6 +108,8 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; + #elif defined (DJGPP) @@ -112,11 +118,13 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; #elif defined(R5900) /* PS2 EE */ typedef long ogg_int64_t; + typedef unsigned long ogg_uint64_t; typedef int ogg_int32_t; typedef unsigned ogg_uint32_t; typedef short ogg_int16_t; @@ -129,6 +137,7 @@ typedef signed int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long int ogg_int64_t; + typedef unsigned long long int ogg_uint64_t; #elif defined(__TMS320C6X__) @@ -138,6 +147,7 @@ typedef signed int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long int ogg_int64_t; + typedef unsigned long long int ogg_uint64_t; #else From 9360b2354486fe5ae75dd3860cf71dd55df7c63c Mon Sep 17 00:00:00 2001 From: PouleyKetchoupp Date: Tue, 12 Nov 2019 08:15:29 +0100 Subject: [PATCH 015/113] Fixed crash when pressing down key on empty Tree Fixes #33554 (cherry picked from commit 47a4ca0022fe961d7dc607f37b0af6b5d0283d2e) --- scene/gui/tree.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scene/gui/tree.cpp b/scene/gui/tree.cpp index b10e16afc62..59e218de6f3 100644 --- a/scene/gui/tree.cpp +++ b/scene/gui/tree.cpp @@ -2152,8 +2152,9 @@ void Tree::_go_down() { TreeItem *next = NULL; if (!selected_item) { - next = hide_root ? root->get_next_visible() : root; - selected_item = 0; + if (root) { + next = hide_root ? root->get_next_visible() : root; + } } else { next = selected_item->get_next_visible(); From 716f7c3d32654477649898706b58c1f476d53e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 07:51:51 +0100 Subject: [PATCH 016/113] X11: Fix memory leak in handle_key_event Extracted from #27189. (cherry picked from commit d191ffb47ddd50df470ef1714c7df6b4da9c7bca) --- platform/x11/os_x11.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp index 58f50f072d9..397e8a25bd4 100644 --- a/platform/x11/os_x11.cpp +++ b/platform/x11/os_x11.cpp @@ -1665,6 +1665,7 @@ void OS_X11::handle_key_event(XKeyEvent *p_event, bool p_echo) { input->accumulate_input_event(k); } + memfree(utf8string); return; } memfree(utf8string); From 699b0872960e58dbb09c30d962150b0c49e14de2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 08:45:51 +0100 Subject: [PATCH 017/113] glad: Sync with upstream 0.1.33 (cherry picked from commit 92c3b6c85ef2268e20a4e072423f5eb902d7acef) --- thirdparty/README.md | 2 +- thirdparty/glad/glad.c | 5 ++++- thirdparty/glad/glad/glad.h | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index ad703bcdff6..76d9af91002 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -131,7 +131,7 @@ Files extracted from upstream source: ## glad - Upstream: https://github.com/Dav1dde/glad -- Version: 0.1.31 +- Version: 0.1.33 - License: MIT The files we package are automatically generated. diff --git a/thirdparty/glad/glad.c b/thirdparty/glad/glad.c index 6bc54a33b06..4159e85ef48 100644 --- a/thirdparty/glad/glad.c +++ b/thirdparty/glad/glad.c @@ -1,6 +1,6 @@ /* - OpenGL loader generated by glad 0.1.31 on Thu Jul 11 10:09:18 2019. + OpenGL loader generated by glad 0.1.33 on Tue Nov 12 08:44:22 2019. Language/Generator: C/C++ Specification: gl @@ -29,6 +29,9 @@ static void* get_proc(const char *namez); #if defined(_WIN32) || defined(__CYGWIN__) +#ifndef _WINDOWS_ +#undef APIENTRY +#endif #include static HMODULE libGL; diff --git a/thirdparty/glad/glad/glad.h b/thirdparty/glad/glad/glad.h index cba90ff9bda..66e90491e29 100644 --- a/thirdparty/glad/glad/glad.h +++ b/thirdparty/glad/glad/glad.h @@ -1,6 +1,6 @@ /* - OpenGL loader generated by glad 0.1.31 on Thu Jul 11 10:09:18 2019. + OpenGL loader generated by glad 0.1.33 on Tue Nov 12 08:44:22 2019. Language/Generator: C/C++ Specification: gl @@ -3402,7 +3402,7 @@ GLAPI PFNGLWAITSYNCPROC glad_glWaitSync; typedef void (APIENTRYP PFNGLGETINTEGER64VPROC)(GLenum pname, GLint64 *data); GLAPI PFNGLGETINTEGER64VPROC glad_glGetInteger64v; #define glGetInteger64v glad_glGetInteger64v -typedef void (APIENTRYP PFNGLGETSYNCIVPROC)(GLsync sync, GLenum pname, GLsizei bufSize, GLsizei *length, GLint *values); +typedef void (APIENTRYP PFNGLGETSYNCIVPROC)(GLsync sync, GLenum pname, GLsizei count, GLsizei *length, GLint *values); GLAPI PFNGLGETSYNCIVPROC glad_glGetSynciv; #define glGetSynciv glad_glGetSynciv typedef void (APIENTRYP PFNGLGETINTEGER64I_VPROC)(GLenum target, GLuint index, GLint64 *data); From 1b7a2e72844b9835d28629a6ac227477d5c3632b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 08:55:43 +0100 Subject: [PATCH 018/113] mbedtls: Update to upstream version 2.16.3 (cherry picked from commit a3d77be53b066e87199d1f91047fa40cb062f77f) --- thirdparty/README.md | 11 +- thirdparty/mbedtls/include/mbedtls/bn_mul.h | 3 +- .../mbedtls/include/mbedtls/check_config.h | 8 +- thirdparty/mbedtls/include/mbedtls/config.h | 94 +- thirdparty/mbedtls/include/mbedtls/ecdsa.h | 59 ++ thirdparty/mbedtls/include/mbedtls/hkdf.h | 24 +- .../mbedtls/include/mbedtls/hmac_drbg.h | 2 +- thirdparty/mbedtls/include/mbedtls/pk.h | 8 + .../mbedtls/include/mbedtls/platform_util.h | 11 + thirdparty/mbedtls/include/mbedtls/rsa.h | 15 +- thirdparty/mbedtls/include/mbedtls/ssl.h | 4 +- .../mbedtls/include/mbedtls/ssl_ticket.h | 4 +- thirdparty/mbedtls/include/mbedtls/version.h | 8 +- thirdparty/mbedtls/include/mbedtls/x509.h | 6 +- thirdparty/mbedtls/include/mbedtls/x509_crl.h | 4 +- thirdparty/mbedtls/library/bignum.c | 14 +- thirdparty/mbedtls/library/certs.c | 999 +++++++++--------- thirdparty/mbedtls/library/ecdsa.c | 125 ++- thirdparty/mbedtls/library/ecjpake.c | 4 +- thirdparty/mbedtls/library/entropy_poll.c | 29 +- thirdparty/mbedtls/library/error.c | 2 +- thirdparty/mbedtls/library/havege.c | 30 +- thirdparty/mbedtls/library/hmac_drbg.c | 86 +- thirdparty/mbedtls/library/net_sockets.c | 2 +- thirdparty/mbedtls/library/pkwrite.c | 27 +- thirdparty/mbedtls/library/platform_util.c | 5 +- thirdparty/mbedtls/library/ssl_srv.c | 2 +- thirdparty/mbedtls/library/ssl_tls.c | 6 +- thirdparty/mbedtls/library/timing.c | 1 - thirdparty/mbedtls/library/version_features.c | 3 + thirdparty/mbedtls/library/x509.c | 2 +- thirdparty/mbedtls/library/x509_crt.c | 44 +- thirdparty/mbedtls/library/x509write_crt.c | 12 +- thirdparty/mbedtls/library/x509write_csr.c | 12 +- thirdparty/mbedtls/{ => patches}/1453.diff | 0 thirdparty/mbedtls/{ => patches}/padlock.diff | 0 36 files changed, 1009 insertions(+), 657 deletions(-) rename thirdparty/mbedtls/{ => patches}/1453.diff (100%) rename thirdparty/mbedtls/{ => patches}/padlock.diff (100%) diff --git a/thirdparty/README.md b/thirdparty/README.md index 76d9af91002..0c29563c889 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -285,14 +285,19 @@ Godot build configurations, check them out when updating. ## mbedtls - Upstream: https://tls.mbed.org/ -- Version: 2.16.2 +- Version: 2.16.3 - License: Apache 2.0 File extracted from upstream release tarball (`-apache.tgz` variant): - All `*.h` from `include/mbedtls/` to `thirdparty/mbedtls/include/mbedtls/` - All `*.c` from `library/` to `thirdparty/mbedtls/library/` -- Applied the patch in `thirdparty/mbedtls/1453.diff` (PR 1453). Soon to be merged upstream. Check it out at next update. -- Applied the patch in `thirdparty/mbedtls/padlock.diff`. This disables VIA padlock support which defines a symbol `unsupported` which clashses with a symbol in libwebsockets. +- LICENSE and apache-2.0.txt files +- Applied the patch in `thirdparty/mbedtls/patches/1453.diff` (PR 1453). + Soon to be merged upstream. Check it out at next update. +- Applied the patch in `thirdparty/mbedtls/patches/padlock.diff`. This disables VIA + padlock support which defines a symbol `unsupported` which clashes with + a symbol in libwebsockets. + ## miniupnpc diff --git a/thirdparty/mbedtls/include/mbedtls/bn_mul.h b/thirdparty/mbedtls/include/mbedtls/bn_mul.h index c33bd8d4ab5..748975ea512 100644 --- a/thirdparty/mbedtls/include/mbedtls/bn_mul.h +++ b/thirdparty/mbedtls/include/mbedtls/bn_mul.h @@ -642,7 +642,8 @@ "r6", "r7", "r8", "r9", "cc" \ ); -#elif defined (__ARM_FEATURE_DSP) && (__ARM_FEATURE_DSP == 1) +#elif (__ARM_ARCH >= 6) && \ + defined (__ARM_FEATURE_DSP) && (__ARM_FEATURE_DSP == 1) #define MULADDC_INIT \ asm( diff --git a/thirdparty/mbedtls/include/mbedtls/check_config.h b/thirdparty/mbedtls/include/mbedtls/check_config.h index b86e5807e00..6eabcc8748f 100644 --- a/thirdparty/mbedtls/include/mbedtls/check_config.h +++ b/thirdparty/mbedtls/include/mbedtls/check_config.h @@ -123,7 +123,7 @@ #error "MBEDTLS_ECDSA_DETERMINISTIC defined, but not all prerequisites" #endif -#if defined(MBEDTLS_ECP_C) && ( !defined(MBEDTLS_BIGNUM_C) || ( \ +#if defined(MBEDTLS_ECP_C) && ( !defined(MBEDTLS_BIGNUM_C) || ( \ !defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) && \ !defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) && \ !defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) && \ @@ -134,7 +134,9 @@ !defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) && \ !defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) && \ !defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) ) ) + !defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) && \ + !defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) && \ + !defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) ) ) #error "MBEDTLS_ECP_C defined, but not all prerequisites" #endif @@ -691,7 +693,7 @@ /* * Avoid warning from -pedantic. This is a convenient place for this * workaround since this is included by every single file before the - * #if defined(MBEDTLS_xxx_C) that results in emtpy translation units. + * #if defined(MBEDTLS_xxx_C) that results in empty translation units. */ typedef int mbedtls_iso_c_forbids_empty_translation_units; diff --git a/thirdparty/mbedtls/include/mbedtls/config.h b/thirdparty/mbedtls/include/mbedtls/config.h index e16e1e53d38..0cc502cd79e 100644 --- a/thirdparty/mbedtls/include/mbedtls/config.h +++ b/thirdparty/mbedtls/include/mbedtls/config.h @@ -139,7 +139,7 @@ * * System has time.h, time(), and an implementation for * mbedtls_platform_gmtime_r() (see below). - * The time needs to be correct (not necesarily very accurate, but at least + * The time needs to be correct (not necessarily very accurate, but at least * the date should be correct). This is used to verify the validity period of * X.509 certificates. * @@ -276,28 +276,52 @@ * For example, when a function accepts as input a pointer to a buffer that may * contain untrusted data, and its documentation mentions that this pointer * must not be NULL: - * - the pointer is checked to be non-NULL only if this option is enabled - * - the content of the buffer is always validated + * - The pointer is checked to be non-NULL only if this option is enabled. + * - The content of the buffer is always validated. * * When this flag is defined, if a library function receives a parameter that - * is invalid, it will: - * - invoke the macro MBEDTLS_PARAM_FAILED() which by default expands to a - * call to the function mbedtls_param_failed() - * - immediately return (with a specific error code unless the function - * returns void and can't communicate an error). + * is invalid: + * 1. The function will invoke the macro MBEDTLS_PARAM_FAILED(). + * 2. If MBEDTLS_PARAM_FAILED() did not terminate the program, the function + * will immediately return. If the function returns an Mbed TLS error code, + * the error code in this case is MBEDTLS_ERR_xxx_BAD_INPUT_DATA. * - * When defining this flag, you also need to: - * - either provide a definition of the function mbedtls_param_failed() in - * your application (see platform_util.h for its prototype) as the library - * calls that function, but does not provide a default definition for it, - * - or provide a different definition of the macro MBEDTLS_PARAM_FAILED() - * below if the above mechanism is not flexible enough to suit your needs. - * See the documentation of this macro later in this file. + * When defining this flag, you also need to arrange a definition for + * MBEDTLS_PARAM_FAILED(). You can do this by any of the following methods: + * - By default, the library defines MBEDTLS_PARAM_FAILED() to call a + * function mbedtls_param_failed(), but the library does not define this + * function. If you do not make any other arrangements, you must provide + * the function mbedtls_param_failed() in your application. + * See `platform_util.h` for its prototype. + * - If you enable the macro #MBEDTLS_CHECK_PARAMS_ASSERT, then the + * library defines #MBEDTLS_PARAM_FAILED(\c cond) to be `assert(cond)`. + * You can still supply an alternative definition of + * MBEDTLS_PARAM_FAILED(), which may call `assert`. + * - If you define a macro MBEDTLS_PARAM_FAILED() before including `config.h` + * or you uncomment the definition of MBEDTLS_PARAM_FAILED() in `config.h`, + * the library will call the macro that you defined and will not supply + * its own version. Note that if MBEDTLS_PARAM_FAILED() calls `assert`, + * you need to enable #MBEDTLS_CHECK_PARAMS_ASSERT so that library source + * files include ``. * * Uncomment to enable validation of application-controlled parameters. */ //#define MBEDTLS_CHECK_PARAMS +/** + * \def MBEDTLS_CHECK_PARAMS_ASSERT + * + * Allow MBEDTLS_PARAM_FAILED() to call `assert`, and make it default to + * `assert`. This macro is only used if #MBEDTLS_CHECK_PARAMS is defined. + * + * If this macro is not defined, then MBEDTLS_PARAM_FAILED() defaults to + * calling a function mbedtls_param_failed(). See the documentation of + * #MBEDTLS_CHECK_PARAMS for details. + * + * Uncomment to allow MBEDTLS_PARAM_FAILED() to call `assert`. + */ +//#define MBEDTLS_CHECK_PARAMS_ASSERT + /* \} name SECTION: System support */ /** @@ -401,7 +425,7 @@ * \note Because of a signature change, the core AES encryption and decryption routines are * currently named mbedtls_aes_internal_encrypt and mbedtls_aes_internal_decrypt, * respectively. When setting up alternative implementations, these functions should - * be overriden, but the wrapper functions mbedtls_aes_decrypt and mbedtls_aes_encrypt + * be overridden, but the wrapper functions mbedtls_aes_decrypt and mbedtls_aes_encrypt * must stay untouched. * * \note If you use the AES_xxx_ALT macros, then is is recommended to also set @@ -416,6 +440,16 @@ * dependencies on them, and considering stronger message digests * and ciphers instead. * + * \warning If both MBEDTLS_ECDSA_SIGN_ALT and MBEDTLS_ECDSA_DETERMINISTIC are + * enabled, then the deterministic ECDH signature functions pass the + * the static HMAC-DRBG as RNG to mbedtls_ecdsa_sign(). Therefore + * alternative implementations should use the RNG only for generating + * the ephemeral key and nothing else. If this is not possible, then + * MBEDTLS_ECDSA_DETERMINISTIC should be disabled and an alternative + * implementation should be provided for mbedtls_ecdsa_sign_det_ext() + * (and for mbedtls_ecdsa_sign_det() too if backward compatibility is + * desirable). + * */ //#define MBEDTLS_MD2_PROCESS_ALT //#define MBEDTLS_MD4_PROCESS_ALT @@ -1558,7 +1592,7 @@ * \def MBEDTLS_SSL_SESSION_TICKETS * * Enable support for RFC 5077 session tickets in SSL. - * Client-side, provides full support for session tickets (maintainance of a + * Client-side, provides full support for session tickets (maintenance of a * session store remains the responsibility of the application, though). * Server-side, you also need to provide callbacks for writing and parsing * tickets, including authenticated encryption and key management. Example @@ -1642,9 +1676,7 @@ * * Uncomment this to enable pthread mutexes. */ -// -- GODOT start -- //#define MBEDTLS_THREADING_PTHREAD -// -- GODOT end -- /** * \def MBEDTLS_VERSION_FEATURES @@ -1726,7 +1758,7 @@ * * \warning TLS-level compression MAY REDUCE SECURITY! See for example the * CRIME attack. Before enabling this option, you should examine with care if - * CRIME or similar exploits may be a applicable to your use case. + * CRIME or similar exploits may be applicable to your use case. * * \note Currently compression can't be used with DTLS. * @@ -2838,9 +2870,7 @@ * * Enable this layer to allow use of mutexes within mbed TLS */ -// -- GODOT start -- //#define MBEDTLS_THREADING_C -// -- GODOT end -- /** * \def MBEDTLS_TIMING_C @@ -3042,7 +3072,7 @@ //#define MBEDTLS_PLATFORM_STD_TIME time /**< Default time to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled */ //#define MBEDTLS_PLATFORM_STD_FPRINTF fprintf /**< Default fprintf to use, can be undefined */ //#define MBEDTLS_PLATFORM_STD_PRINTF printf /**< Default printf to use, can be undefined */ -/* Note: your snprintf must correclty zero-terminate the buffer! */ +/* Note: your snprintf must correctly zero-terminate the buffer! */ //#define MBEDTLS_PLATFORM_STD_SNPRINTF snprintf /**< Default snprintf to use, can be undefined */ //#define MBEDTLS_PLATFORM_STD_EXIT_SUCCESS 0 /**< Default exit value to use, can be undefined */ //#define MBEDTLS_PLATFORM_STD_EXIT_FAILURE 1 /**< Default exit value to use, can be undefined */ @@ -3059,20 +3089,23 @@ //#define MBEDTLS_PLATFORM_TIME_TYPE_MACRO time_t /**< Default time macro to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled */ //#define MBEDTLS_PLATFORM_FPRINTF_MACRO fprintf /**< Default fprintf macro to use, can be undefined */ //#define MBEDTLS_PLATFORM_PRINTF_MACRO printf /**< Default printf macro to use, can be undefined */ -/* Note: your snprintf must correclty zero-terminate the buffer! */ +/* Note: your snprintf must correctly zero-terminate the buffer! */ //#define MBEDTLS_PLATFORM_SNPRINTF_MACRO snprintf /**< Default snprintf macro to use, can be undefined */ //#define MBEDTLS_PLATFORM_NV_SEED_READ_MACRO mbedtls_platform_std_nv_seed_read /**< Default nv_seed_read function to use, can be undefined */ //#define MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO mbedtls_platform_std_nv_seed_write /**< Default nv_seed_write function to use, can be undefined */ /** * \brief This macro is invoked by the library when an invalid parameter - * is detected that is only checked with MBEDTLS_CHECK_PARAMS + * is detected that is only checked with #MBEDTLS_CHECK_PARAMS * (see the documentation of that option for context). * - * When you leave this undefined here, a default definition is - * provided that invokes the function mbedtls_param_failed(), - * which is declared in platform_util.h for the benefit of the - * library, but that you need to define in your application. + * When you leave this undefined here, the library provides + * a default definition. If the macro #MBEDTLS_CHECK_PARAMS_ASSERT + * is defined, the default definition is `assert(cond)`, + * otherwise the default definition calls a function + * mbedtls_param_failed(). This function is declared in + * `platform_util.h` for the benefit of the library, but + * you need to define in your application. * * When you define this here, this replaces the default * definition in platform_util.h (which no longer declares the @@ -3081,6 +3114,9 @@ * particular, that all the necessary declarations are visible * from within the library - you can ensure that by providing * them in this file next to the macro definition). + * If you define this macro to call `assert`, also define + * #MBEDTLS_CHECK_PARAMS_ASSERT so that library source files + * include ``. * * Note that you may define this macro to expand to nothing, in * which case you don't have to worry about declarations or diff --git a/thirdparty/mbedtls/include/mbedtls/ecdsa.h b/thirdparty/mbedtls/include/mbedtls/ecdsa.h index f8b28507c21..932acc6d141 100644 --- a/thirdparty/mbedtls/include/mbedtls/ecdsa.h +++ b/thirdparty/mbedtls/include/mbedtls/ecdsa.h @@ -175,6 +175,19 @@ int mbedtls_ecdsa_sign( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, * (SECG): SEC1 Elliptic Curve Cryptography, section * 4.1.3, step 5. * + * \warning Since the output of the internal RNG is always the same for + * the same key and message, this limits the efficiency of + * blinding and leaks information through side channels. For + * secure behavior use mbedtls_ecdsa_sign_det_ext() instead. + * + * (Optimally the blinding is a random value that is different + * on every execution. In this case the blinding is still + * random from the attackers perspective, but is the same on + * each execution. This means that this blinding does not + * prevent attackers from recovering secrets by combining + * several measurement traces, but may prevent some attacks + * that exploit relationships between secret data.) + * * \see ecp.h * * \param grp The context for the elliptic curve to use. @@ -200,6 +213,52 @@ int mbedtls_ecdsa_sign_det( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, const mbedtls_mpi *d, const unsigned char *buf, size_t blen, mbedtls_md_type_t md_alg ); +/** + * \brief This function computes the ECDSA signature of a + * previously-hashed message, deterministic version. + * + * For more information, see RFC-6979: Deterministic + * Usage of the Digital Signature Algorithm (DSA) and Elliptic + * Curve Digital Signature Algorithm (ECDSA). + * + * \note If the bitlength of the message hash is larger than the + * bitlength of the group order, then the hash is truncated as + * defined in Standards for Efficient Cryptography Group + * (SECG): SEC1 Elliptic Curve Cryptography, section + * 4.1.3, step 5. + * + * \see ecp.h + * + * \param grp The context for the elliptic curve to use. + * This must be initialized and have group parameters + * set, for example through mbedtls_ecp_group_load(). + * \param r The MPI context in which to store the first part + * the signature. This must be initialized. + * \param s The MPI context in which to store the second part + * the signature. This must be initialized. + * \param d The private signing key. This must be initialized + * and setup, for example through mbedtls_ecp_gen_privkey(). + * \param buf The hashed content to be signed. This must be a readable + * buffer of length \p blen Bytes. It may be \c NULL if + * \p blen is zero. + * \param blen The length of \p buf in Bytes. + * \param md_alg The hash algorithm used to hash the original data. + * \param f_rng_blind The RNG function used for blinding. This must not be + * \c NULL. + * \param p_rng_blind The RNG context to be passed to \p f_rng. This may be + * \c NULL if \p f_rng doesn't need a context parameter. + * + * \return \c 0 on success. + * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX + * error code on failure. + */ +int mbedtls_ecdsa_sign_det_ext( mbedtls_ecp_group *grp, mbedtls_mpi *r, + mbedtls_mpi *s, const mbedtls_mpi *d, + const unsigned char *buf, size_t blen, + mbedtls_md_type_t md_alg, + int (*f_rng_blind)(void *, unsigned char *, + size_t), + void *p_rng_blind ); #endif /* MBEDTLS_ECDSA_DETERMINISTIC */ /** diff --git a/thirdparty/mbedtls/include/mbedtls/hkdf.h b/thirdparty/mbedtls/include/mbedtls/hkdf.h index 40ee64eb03c..bcafe425136 100644 --- a/thirdparty/mbedtls/include/mbedtls/hkdf.h +++ b/thirdparty/mbedtls/include/mbedtls/hkdf.h @@ -7,22 +7,22 @@ * specified by RFC 5869. */ /* - * Copyright (C) 2016-2018, ARM Limited, All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 + * Copyright (C) 2016-2019, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * - * This file is part of mbed TLS (https://tls.mbed.org) + * This file is part of mbed TLS (https://tls.mbed.org) */ #ifndef MBEDTLS_HKDF_H #define MBEDTLS_HKDF_H diff --git a/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h b/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h index 7eae32bbd67..f1289cb306c 100644 --- a/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h +++ b/thirdparty/mbedtls/include/mbedtls/hmac_drbg.h @@ -82,7 +82,7 @@ extern "C" { */ typedef struct mbedtls_hmac_drbg_context { - /* Working state: the key K is not stored explicitely, + /* Working state: the key K is not stored explicitly, * but is implied by the HMAC context */ mbedtls_md_context_t md_ctx; /*!< HMAC context (inc. K) */ unsigned char V[MBEDTLS_MD_MAX_SIZE]; /*!< V in the spec */ diff --git a/thirdparty/mbedtls/include/mbedtls/pk.h b/thirdparty/mbedtls/include/mbedtls/pk.h index 91950f9407b..136427503a0 100644 --- a/thirdparty/mbedtls/include/mbedtls/pk.h +++ b/thirdparty/mbedtls/include/mbedtls/pk.h @@ -416,6 +416,10 @@ int mbedtls_pk_verify_ext( mbedtls_pk_type_t type, const void *options, * * \note For RSA, md_alg may be MBEDTLS_MD_NONE if hash_len != 0. * For ECDSA, md_alg may never be MBEDTLS_MD_NONE. + * + * \note In order to ensure enough space for the signature, the + * \p sig buffer size must be of at least + * `max(MBEDTLS_ECDSA_MAX_LEN, MBEDTLS_MPI_MAX_SIZE)` bytes. */ int mbedtls_pk_sign( mbedtls_pk_context *ctx, mbedtls_md_type_t md_alg, const unsigned char *hash, size_t hash_len, @@ -430,6 +434,10 @@ int mbedtls_pk_sign( mbedtls_pk_context *ctx, mbedtls_md_type_t md_alg, * \c mbedtls_ecp_set_max_ops() to reduce blocking for ECC * operations. For RSA, same as \c mbedtls_pk_sign(). * + * \note In order to ensure enough space for the signature, the + * \p sig buffer size must be of at least + * `max(MBEDTLS_ECDSA_MAX_LEN, MBEDTLS_MPI_MAX_SIZE)` bytes. + * * \param ctx The PK context to use. It must have been set up * with a private key. * \param md_alg Hash algorithm used (see notes) diff --git a/thirdparty/mbedtls/include/mbedtls/platform_util.h b/thirdparty/mbedtls/include/mbedtls/platform_util.h index dba6d459822..09d09651829 100644 --- a/thirdparty/mbedtls/include/mbedtls/platform_util.h +++ b/thirdparty/mbedtls/include/mbedtls/platform_util.h @@ -43,6 +43,12 @@ extern "C" { #if defined(MBEDTLS_CHECK_PARAMS) +#if defined(MBEDTLS_CHECK_PARAMS_ASSERT) +/* Allow the user to define MBEDTLS_PARAM_FAILED to something like assert + * (which is what our config.h suggests). */ +#include +#endif /* MBEDTLS_CHECK_PARAMS_ASSERT */ + #if defined(MBEDTLS_PARAM_FAILED) /** An alternative definition of MBEDTLS_PARAM_FAILED has been set in config.h. * @@ -50,6 +56,11 @@ extern "C" { * MBEDTLS_PARAM_FAILED() will expand to a call to mbedtls_param_failed(). */ #define MBEDTLS_PARAM_FAILED_ALT + +#elif defined(MBEDTLS_CHECK_PARAMS_ASSERT) +#define MBEDTLS_PARAM_FAILED( cond ) assert( cond ) +#define MBEDTLS_PARAM_FAILED_ALT + #else /* MBEDTLS_PARAM_FAILED */ #define MBEDTLS_PARAM_FAILED( cond ) \ mbedtls_param_failed( #cond, __FILE__, __LINE__ ) diff --git a/thirdparty/mbedtls/include/mbedtls/rsa.h b/thirdparty/mbedtls/include/mbedtls/rsa.h index 906c4273325..35bacd8763c 100644 --- a/thirdparty/mbedtls/include/mbedtls/rsa.h +++ b/thirdparty/mbedtls/include/mbedtls/rsa.h @@ -150,13 +150,13 @@ mbedtls_rsa_context; * \note The choice of padding mode is strictly enforced for private key * operations, since there might be security concerns in * mixing padding modes. For public key operations it is - * a default value, which can be overriden by calling specific + * a default value, which can be overridden by calling specific * \c rsa_rsaes_xxx or \c rsa_rsassa_xxx functions. * * \note The hash selected in \p hash_id is always used for OEAP * encryption. For PSS signatures, it is always used for - * making signatures, but can be overriden for verifying them. - * If set to #MBEDTLS_MD_NONE, it is always overriden. + * making signatures, but can be overridden for verifying them. + * If set to #MBEDTLS_MD_NONE, it is always overridden. * * \param ctx The RSA context to initialize. This must not be \c NULL. * \param padding The padding mode to use. This must be either @@ -904,7 +904,8 @@ int mbedtls_rsa_rsaes_oaep_decrypt( mbedtls_rsa_context *ctx, * the size of the hash corresponding to \p md_alg. * \param sig The buffer to hold the signature. This must be a writable * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. + * for an 2048-bit RSA modulus. A buffer length of + * #MBEDTLS_MPI_MAX_SIZE is always safe. * * \return \c 0 if the signing operation was successful. * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. @@ -951,7 +952,8 @@ int mbedtls_rsa_pkcs1_sign( mbedtls_rsa_context *ctx, * the size of the hash corresponding to \p md_alg. * \param sig The buffer to hold the signature. This must be a writable * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. + * for an 2048-bit RSA modulus. A buffer length of + * #MBEDTLS_MPI_MAX_SIZE is always safe. * * \return \c 0 if the signing operation was successful. * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. @@ -1012,7 +1014,8 @@ int mbedtls_rsa_rsassa_pkcs1_v15_sign( mbedtls_rsa_context *ctx, * the size of the hash corresponding to \p md_alg. * \param sig The buffer to hold the signature. This must be a writable * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. + * for an 2048-bit RSA modulus. A buffer length of + * #MBEDTLS_MPI_MAX_SIZE is always safe. * * \return \c 0 if the signing operation was successful. * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. diff --git a/thirdparty/mbedtls/include/mbedtls/ssl.h b/thirdparty/mbedtls/include/mbedtls/ssl.h index d31f6cdd566..1adf9608ccf 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl.h @@ -2033,7 +2033,7 @@ void mbedtls_ssl_conf_ca_chain( mbedtls_ssl_config *conf, * provision more than one cert/key pair (eg one ECDSA, one * RSA with SHA-256, one RSA with SHA-1). An adequate * certificate will be selected according to the client's - * advertised capabilities. In case mutliple certificates are + * advertised capabilities. In case multiple certificates are * adequate, preference is given to the one set by the first * call to this function, then second, etc. * @@ -3206,7 +3206,7 @@ void mbedtls_ssl_free( mbedtls_ssl_context *ssl ); * mbedtls_ssl_config_defaults() or mbedtls_ssl_config_free(). * * \note You need to call mbedtls_ssl_config_defaults() unless you - * manually set all of the relevent fields yourself. + * manually set all of the relevant fields yourself. * * \param conf SSL configuration context */ diff --git a/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h b/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h index a84e7816e4a..774a007a9f3 100644 --- a/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h +++ b/thirdparty/mbedtls/include/mbedtls/ssl_ticket.h @@ -117,14 +117,14 @@ int mbedtls_ssl_ticket_setup( mbedtls_ssl_ticket_context *ctx, /** * \brief Implementation of the ticket write callback * - * \note See \c mbedlts_ssl_ticket_write_t for description + * \note See \c mbedtls_ssl_ticket_write_t for description */ mbedtls_ssl_ticket_write_t mbedtls_ssl_ticket_write; /** * \brief Implementation of the ticket parse callback * - * \note See \c mbedlts_ssl_ticket_parse_t for description + * \note See \c mbedtls_ssl_ticket_parse_t for description */ mbedtls_ssl_ticket_parse_t mbedtls_ssl_ticket_parse; diff --git a/thirdparty/mbedtls/include/mbedtls/version.h b/thirdparty/mbedtls/include/mbedtls/version.h index ef8e4c1f4fe..b4eef71e509 100644 --- a/thirdparty/mbedtls/include/mbedtls/version.h +++ b/thirdparty/mbedtls/include/mbedtls/version.h @@ -40,16 +40,16 @@ */ #define MBEDTLS_VERSION_MAJOR 2 #define MBEDTLS_VERSION_MINOR 16 -#define MBEDTLS_VERSION_PATCH 2 +#define MBEDTLS_VERSION_PATCH 3 /** * The single version number has the following structure: * MMNNPP00 * Major version | Minor version | Patch version */ -#define MBEDTLS_VERSION_NUMBER 0x02100200 -#define MBEDTLS_VERSION_STRING "2.16.2" -#define MBEDTLS_VERSION_STRING_FULL "mbed TLS 2.16.2" +#define MBEDTLS_VERSION_NUMBER 0x02100300 +#define MBEDTLS_VERSION_STRING "2.16.3" +#define MBEDTLS_VERSION_STRING_FULL "mbed TLS 2.16.3" #if defined(MBEDTLS_VERSION_C) diff --git a/thirdparty/mbedtls/include/mbedtls/x509.h b/thirdparty/mbedtls/include/mbedtls/x509.h index 9ae825c1838..63aae32d870 100644 --- a/thirdparty/mbedtls/include/mbedtls/x509.h +++ b/thirdparty/mbedtls/include/mbedtls/x509.h @@ -77,7 +77,7 @@ #define MBEDTLS_ERR_X509_ALLOC_FAILED -0x2880 /**< Allocation of memory failed. */ #define MBEDTLS_ERR_X509_FILE_IO_ERROR -0x2900 /**< Read/write of file failed. */ #define MBEDTLS_ERR_X509_BUFFER_TOO_SMALL -0x2980 /**< Destination buffer is too small. */ -#define MBEDTLS_ERR_X509_FATAL_ERROR -0x3000 /**< A fatal error occured, eg the chain is too long or the vrfy callback failed. */ +#define MBEDTLS_ERR_X509_FATAL_ERROR -0x3000 /**< A fatal error occurred, eg the chain is too long or the vrfy callback failed. */ /* \} name */ /** @@ -250,7 +250,7 @@ int mbedtls_x509_serial_gets( char *buf, size_t size, const mbedtls_x509_buf *se * * \param to mbedtls_x509_time to check * - * \return 1 if the given time is in the past or an error occured, + * \return 1 if the given time is in the past or an error occurred, * 0 otherwise. */ int mbedtls_x509_time_is_past( const mbedtls_x509_time *to ); @@ -264,7 +264,7 @@ int mbedtls_x509_time_is_past( const mbedtls_x509_time *to ); * * \param from mbedtls_x509_time to check * - * \return 1 if the given time is in the future or an error occured, + * \return 1 if the given time is in the future or an error occurred, * 0 otherwise. */ int mbedtls_x509_time_is_future( const mbedtls_x509_time *from ); diff --git a/thirdparty/mbedtls/include/mbedtls/x509_crl.h b/thirdparty/mbedtls/include/mbedtls/x509_crl.h index 08a4283a674..fa838d68cbf 100644 --- a/thirdparty/mbedtls/include/mbedtls/x509_crl.h +++ b/thirdparty/mbedtls/include/mbedtls/x509_crl.h @@ -111,7 +111,7 @@ int mbedtls_x509_crl_parse_der( mbedtls_x509_crl *chain, /** * \brief Parse one or more CRLs and append them to the chained list * - * \note Mutliple CRLs are accepted only if using PEM format + * \note Multiple CRLs are accepted only if using PEM format * * \param chain points to the start of the chain * \param buf buffer holding the CRL data in PEM or DER format @@ -126,7 +126,7 @@ int mbedtls_x509_crl_parse( mbedtls_x509_crl *chain, const unsigned char *buf, s /** * \brief Load one or more CRLs and append them to the chained list * - * \note Mutliple CRLs are accepted only if using PEM format + * \note Multiple CRLs are accepted only if using PEM format * * \param chain points to the start of the chain * \param path filename to read the CRLs from (in PEM or DER encoding) diff --git a/thirdparty/mbedtls/library/bignum.c b/thirdparty/mbedtls/library/bignum.c index 41946183c57..d1717e94357 100644 --- a/thirdparty/mbedtls/library/bignum.c +++ b/thirdparty/mbedtls/library/bignum.c @@ -742,10 +742,15 @@ cleanup: static mbedtls_mpi_uint mpi_uint_bigendian_to_host_c( mbedtls_mpi_uint x ) { uint8_t i; + unsigned char *x_ptr; mbedtls_mpi_uint tmp = 0; - /* This works regardless of the endianness. */ - for( i = 0; i < ciL; i++, x >>= 8 ) - tmp |= ( x & 0xFF ) << ( ( ciL - 1 - i ) << 3 ); + + for( i = 0, x_ptr = (unsigned char*) &x; i < ciL; i++, x_ptr++ ) + { + tmp <<= CHAR_BIT; + tmp |= (mbedtls_mpi_uint) *x_ptr; + } + return( tmp ); } @@ -2351,7 +2356,8 @@ static int mpi_miller_rabin( const mbedtls_mpi *X, size_t rounds, } if (count++ > 30) { - return MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + ret = MBEDTLS_ERR_MPI_NOT_ACCEPTABLE; + goto cleanup; } } while ( mbedtls_mpi_cmp_mpi( &A, &W ) >= 0 || diff --git a/thirdparty/mbedtls/library/certs.c b/thirdparty/mbedtls/library/certs.c index b07fd8a3a1f..80ab0b9d6c9 100644 --- a/thirdparty/mbedtls/library/certs.c +++ b/thirdparty/mbedtls/library/certs.c @@ -46,75 +46,67 @@ /* BEGIN FILE string macro TEST_CA_CRT_EC_PEM tests/data_files/test-ca2.crt */ #define TEST_CA_CRT_EC_PEM \ "-----BEGIN CERTIFICATE-----\r\n" \ - "MIICUjCCAdegAwIBAgIJAMFD4n5iQ8zoMAoGCCqGSM49BAMCMD4xCzAJBgNVBAYT\r\n" \ - "Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ - "QyBDQTAeFw0xMzA5MjQxNTQ5NDhaFw0yMzA5MjIxNTQ5NDhaMD4xCzAJBgNVBAYT\r\n" \ - "Ak5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UEAxMTUG9sYXJzc2wgVGVzdCBF\r\n" \ - "QyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABMPaKzRBN1gvh1b+/Im6KUNLTuBu\r\n" \ - "ww5XUzM5WNRStJGVOQsj318XJGJI/BqVKc4sLYfCiFKAr9ZqqyHduNMcbli4yuiy\r\n" \ - "aY7zQa0pw7RfdadHb9UZKVVpmlM7ILRmFmAzHqOBoDCBnTAdBgNVHQ4EFgQUnW0g\r\n" \ - "JEkBPyvLeLUZvH4kydv7NnwwbgYDVR0jBGcwZYAUnW0gJEkBPyvLeLUZvH4kydv7\r\n" \ - "NnyhQqRAMD4xCzAJBgNVBAYTAk5MMREwDwYDVQQKEwhQb2xhclNTTDEcMBoGA1UE\r\n" \ - "AxMTUG9sYXJzc2wgVGVzdCBFQyBDQYIJAMFD4n5iQ8zoMAwGA1UdEwQFMAMBAf8w\r\n" \ - "CgYIKoZIzj0EAwIDaQAwZgIxAMO0YnNWKJUAfXgSJtJxexn4ipg+kv4znuR50v56\r\n" \ - "t4d0PCu412mUC6Nnd7izvtE2MgIxAP1nnJQjZ8BWukszFQDG48wxCCyci9qpdSMv\r\n" \ - "uCjn8pwUOkABXK8Mss90fzCfCEOtIA==\r\n" \ + "MIICBDCCAYigAwIBAgIJAMFD4n5iQ8zoMAwGCCqGSM49BAMCBQAwPjELMAkGA1UE\r\n" \ + "BhMCTkwxETAPBgNVBAoMCFBvbGFyU1NMMRwwGgYDVQQDDBNQb2xhcnNzbCBUZXN0\r\n" \ + "IEVDIENBMB4XDTE5MDIxMDE0NDQwMFoXDTI5MDIxMDE0NDQwMFowPjELMAkGA1UE\r\n" \ + "BhMCTkwxETAPBgNVBAoMCFBvbGFyU1NMMRwwGgYDVQQDDBNQb2xhcnNzbCBUZXN0\r\n" \ + "IEVDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEw9orNEE3WC+HVv78ibopQ0tO\r\n" \ + "4G7DDldTMzlY1FK0kZU5CyPfXxckYkj8GpUpziwth8KIUoCv1mqrId240xxuWLjK\r\n" \ + "6LJpjvNBrSnDtF91p0dv1RkpVWmaUzsgtGYWYDMeo1AwTjAMBgNVHRMEBTADAQH/\r\n" \ + "MB0GA1UdDgQWBBSdbSAkSQE/K8t4tRm8fiTJ2/s2fDAfBgNVHSMEGDAWgBSdbSAk\r\n" \ + "SQE/K8t4tRm8fiTJ2/s2fDAMBggqhkjOPQQDAgUAA2gAMGUCMFHKrjAPpHB0BN1a\r\n" \ + "LH8TwcJ3vh0AxeKZj30mRdOKBmg/jLS3rU3g8VQBHpn8sOTTBwIxANxPO5AerimZ\r\n" \ + "hCjMe0d4CTHf1gFZMF70+IqEP+o5VHsIp2Cqvflb0VGWFC5l9a4cQg==\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ /* This is generated from tests/data_files/test-ca2.crt.der using `xxd -i`. */ /* BEGIN FILE binary macro TEST_CA_CRT_EC_DER tests/data_files/test-ca2.crt.der */ #define TEST_CA_CRT_EC_DER { \ - 0x30, 0x82, 0x02, 0x52, 0x30, 0x82, 0x01, 0xd7, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x09, 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, \ - 0x30, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, \ - 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, \ - 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, \ - 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, \ - 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, \ - 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, 0x31, 0x33, 0x30, 0x39, \ - 0x32, 0x34, 0x31, 0x35, 0x34, 0x39, 0x34, 0x38, 0x5a, 0x17, 0x0d, 0x32, \ - 0x33, 0x30, 0x39, 0x32, 0x32, 0x31, 0x35, 0x34, 0x39, 0x34, 0x38, 0x5a, \ - 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, \ - 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, \ - 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, \ - 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, \ - 0x43, 0x20, 0x43, 0x41, 0x30, 0x76, 0x30, 0x10, 0x06, 0x07, 0x2a, 0x86, \ - 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, 0x05, 0x2b, 0x81, 0x04, 0x00, 0x22, \ - 0x03, 0x62, 0x00, 0x04, 0xc3, 0xda, 0x2b, 0x34, 0x41, 0x37, 0x58, 0x2f, \ - 0x87, 0x56, 0xfe, 0xfc, 0x89, 0xba, 0x29, 0x43, 0x4b, 0x4e, 0xe0, 0x6e, \ - 0xc3, 0x0e, 0x57, 0x53, 0x33, 0x39, 0x58, 0xd4, 0x52, 0xb4, 0x91, 0x95, \ - 0x39, 0x0b, 0x23, 0xdf, 0x5f, 0x17, 0x24, 0x62, 0x48, 0xfc, 0x1a, 0x95, \ - 0x29, 0xce, 0x2c, 0x2d, 0x87, 0xc2, 0x88, 0x52, 0x80, 0xaf, 0xd6, 0x6a, \ - 0xab, 0x21, 0xdd, 0xb8, 0xd3, 0x1c, 0x6e, 0x58, 0xb8, 0xca, 0xe8, 0xb2, \ - 0x69, 0x8e, 0xf3, 0x41, 0xad, 0x29, 0xc3, 0xb4, 0x5f, 0x75, 0xa7, 0x47, \ - 0x6f, 0xd5, 0x19, 0x29, 0x55, 0x69, 0x9a, 0x53, 0x3b, 0x20, 0xb4, 0x66, \ - 0x16, 0x60, 0x33, 0x1e, 0xa3, 0x81, 0xa0, 0x30, 0x81, 0x9d, 0x30, 0x1d, \ - 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x9d, 0x6d, 0x20, \ - 0x24, 0x49, 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, \ - 0xc9, 0xdb, 0xfb, 0x36, 0x7c, 0x30, 0x6e, 0x06, 0x03, 0x55, 0x1d, 0x23, \ - 0x04, 0x67, 0x30, 0x65, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, 0x01, \ - 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, 0xfb, \ - 0x36, 0x7c, 0xa1, 0x42, 0xa4, 0x40, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, \ - 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, \ - 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, \ - 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, \ - 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, \ - 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x82, 0x09, \ - 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, 0x30, 0x0c, 0x06, \ - 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, \ - 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, 0x03, \ - 0x69, 0x00, 0x30, 0x66, 0x02, 0x31, 0x00, 0xc3, 0xb4, 0x62, 0x73, 0x56, \ - 0x28, 0x95, 0x00, 0x7d, 0x78, 0x12, 0x26, 0xd2, 0x71, 0x7b, 0x19, 0xf8, \ - 0x8a, 0x98, 0x3e, 0x92, 0xfe, 0x33, 0x9e, 0xe4, 0x79, 0xd2, 0xfe, 0x7a, \ - 0xb7, 0x87, 0x74, 0x3c, 0x2b, 0xb8, 0xd7, 0x69, 0x94, 0x0b, 0xa3, 0x67, \ - 0x77, 0xb8, 0xb3, 0xbe, 0xd1, 0x36, 0x32, 0x02, 0x31, 0x00, 0xfd, 0x67, \ - 0x9c, 0x94, 0x23, 0x67, 0xc0, 0x56, 0xba, 0x4b, 0x33, 0x15, 0x00, 0xc6, \ - 0xe3, 0xcc, 0x31, 0x08, 0x2c, 0x9c, 0x8b, 0xda, 0xa9, 0x75, 0x23, 0x2f, \ - 0xb8, 0x28, 0xe7, 0xf2, 0x9c, 0x14, 0x3a, 0x40, 0x01, 0x5c, 0xaf, 0x0c, \ - 0xb2, 0xcf, 0x74, 0x7f, 0x30, 0x9f, 0x08, 0x43, 0xad, 0x20 \ + 0x30, 0x82, 0x02, 0x04, 0x30, 0x82, 0x01, 0x88, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x09, 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, \ + 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, \ + 0x05, 0x00, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, \ + 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, \ + 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, 0x13, 0x50, \ + 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, \ + 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, 0x31, 0x39, \ + 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, 0x5a, 0x17, \ + 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, \ + 0x30, 0x5a, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x04, \ + 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, 0x03, 0x55, \ + 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, 0x13, 0x50, \ + 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, 0x73, 0x74, \ + 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x76, 0x30, 0x10, 0x06, 0x07, \ + 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, 0x06, 0x05, 0x2b, 0x81, 0x04, \ + 0x00, 0x22, 0x03, 0x62, 0x00, 0x04, 0xc3, 0xda, 0x2b, 0x34, 0x41, 0x37, \ + 0x58, 0x2f, 0x87, 0x56, 0xfe, 0xfc, 0x89, 0xba, 0x29, 0x43, 0x4b, 0x4e, \ + 0xe0, 0x6e, 0xc3, 0x0e, 0x57, 0x53, 0x33, 0x39, 0x58, 0xd4, 0x52, 0xb4, \ + 0x91, 0x95, 0x39, 0x0b, 0x23, 0xdf, 0x5f, 0x17, 0x24, 0x62, 0x48, 0xfc, \ + 0x1a, 0x95, 0x29, 0xce, 0x2c, 0x2d, 0x87, 0xc2, 0x88, 0x52, 0x80, 0xaf, \ + 0xd6, 0x6a, 0xab, 0x21, 0xdd, 0xb8, 0xd3, 0x1c, 0x6e, 0x58, 0xb8, 0xca, \ + 0xe8, 0xb2, 0x69, 0x8e, 0xf3, 0x41, 0xad, 0x29, 0xc3, 0xb4, 0x5f, 0x75, \ + 0xa7, 0x47, 0x6f, 0xd5, 0x19, 0x29, 0x55, 0x69, 0x9a, 0x53, 0x3b, 0x20, \ + 0xb4, 0x66, 0x16, 0x60, 0x33, 0x1e, 0xa3, 0x50, 0x30, 0x4e, 0x30, 0x0c, \ + 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, 0x30, 0x03, 0x01, 0x01, 0xff, \ + 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x9d, \ + 0x6d, 0x20, 0x24, 0x49, 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, \ + 0x7e, 0x24, 0xc9, 0xdb, 0xfb, 0x36, 0x7c, 0x30, 0x1f, 0x06, 0x03, 0x55, \ + 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, \ + 0x49, 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, \ + 0xdb, 0xfb, 0x36, 0x7c, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, \ + 0x3d, 0x04, 0x03, 0x02, 0x05, 0x00, 0x03, 0x68, 0x00, 0x30, 0x65, 0x02, \ + 0x30, 0x51, 0xca, 0xae, 0x30, 0x0f, 0xa4, 0x70, 0x74, 0x04, 0xdd, 0x5a, \ + 0x2c, 0x7f, 0x13, 0xc1, 0xc2, 0x77, 0xbe, 0x1d, 0x00, 0xc5, 0xe2, 0x99, \ + 0x8f, 0x7d, 0x26, 0x45, 0xd3, 0x8a, 0x06, 0x68, 0x3f, 0x8c, 0xb4, 0xb7, \ + 0xad, 0x4d, 0xe0, 0xf1, 0x54, 0x01, 0x1e, 0x99, 0xfc, 0xb0, 0xe4, 0xd3, \ + 0x07, 0x02, 0x31, 0x00, 0xdc, 0x4f, 0x3b, 0x90, 0x1e, 0xae, 0x29, 0x99, \ + 0x84, 0x28, 0xcc, 0x7b, 0x47, 0x78, 0x09, 0x31, 0xdf, 0xd6, 0x01, 0x59, \ + 0x30, 0x5e, 0xf4, 0xf8, 0x8a, 0x84, 0x3f, 0xea, 0x39, 0x54, 0x7b, 0x08, \ + 0xa7, 0x60, 0xaa, 0xbd, 0xf9, 0x5b, 0xd1, 0x51, 0x96, 0x14, 0x2e, 0x65, \ + 0xf5, 0xae, 0x1c, 0x42 \ } /* END FILE */ @@ -160,7 +152,7 @@ "-----BEGIN CERTIFICATE-----\r\n" \ "MIIDQTCCAimgAwIBAgIBAzANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ - "MTEwMjEyMTQ0NDAwWhcNMjEwMjEyMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "MTkwMjEwMTQ0NDAwWhcNMjkwMjEwMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ "A1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ "CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ "mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ @@ -170,12 +162,12 @@ "KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ "UDBOMAwGA1UdEwQFMAMBAf8wHQYDVR0OBBYEFLRa5KWz3tJS9rnVppUP6z68x/3/\r\n" \ "MB8GA1UdIwQYMBaAFLRa5KWz3tJS9rnVppUP6z68x/3/MA0GCSqGSIb3DQEBCwUA\r\n" \ - "A4IBAQB2W2dIy4q4KysbrTL4HIaOqu62RceGuQ/KhyiI6O0ndCtQ/PgCBqHHTP8u\r\n" \ - "8F1X2ivb60ynHV6baMLPI4Kf1k4MONtLSf/++1qh0Gdycd3A8IDAfy0YnC1F3OPK\r\n" \ - "vWO/cZGitKoTbEpP4y4Rng3sFCDndRCWIRIDOEEW/H3lCcfL7sOQojdLl85ajFkh\r\n" \ - "YvcDqjmnTcspUnuq9Y00C7porXJthZwz1S18qVjcFNk0zEhVMUbupSrdXVmKtOJW\r\n" \ - "MWZjgcA+OXzcnb2hSKWbhjykH/u6/PqkuHPkD723rwXbmHdxRVS9CW57kDkn5ezJ\r\n" \ - "5pE6Sam4qFsCNFJNBV9FRf3ZBMFi\r\n" \ + "A4IBAQA4qFSCth2q22uJIdE4KGHJsJjVEfw2/xn+MkTvCMfxVrvmRvqCtjE4tKDl\r\n" \ + "oK4MxFOek07oDZwvtAT9ijn1hHftTNS7RH9zd/fxNpfcHnMZXVC4w4DNA1fSANtW\r\n" \ + "5sY1JB5Je9jScrsLSS+mAjyv0Ow3Hb2Bix8wu7xNNrV5fIf7Ubm+wt6SqEBxu3Kb\r\n" \ + "+EfObAT4huf3czznhH3C17ed6NSbXwoXfby7stWUDeRJv08RaFOykf/Aae7bY5PL\r\n" \ + "yTVrkAnikMntJ9YI+hNNYt3inqq11A5cN0+rVTst8UKCxzQ4GpvroSwPKTFkbMw4\r\n" \ + "/anT1dVxr/BtwJfiESoK3/4CeXR1\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ @@ -183,76 +175,76 @@ * using `xxd -i`. */ /* BEGIN FILE binary macro TEST_CA_CRT_RSA_SHA256_DER tests/data_files/test-ca-sha256.crt.der */ #define TEST_CA_CRT_RSA_SHA256_DER { \ - 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ - 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ - 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ - 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ - 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ - 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ - 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ - 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ - 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ - 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ - 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ - 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ - 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ - 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ - 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ - 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ - 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ - 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ - 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ - 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ - 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ - 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ - 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ - 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ - 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ - 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ - 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ - 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ - 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ - 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ - 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ - 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ - 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ - 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ - 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ - 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ - 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ - 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, \ - 0x03, 0x82, 0x01, 0x01, 0x00, 0x76, 0x5b, 0x67, 0x48, 0xcb, 0x8a, 0xb8, \ - 0x2b, 0x2b, 0x1b, 0xad, 0x32, 0xf8, 0x1c, 0x86, 0x8e, 0xaa, 0xee, 0xb6, \ - 0x45, 0xc7, 0x86, 0xb9, 0x0f, 0xca, 0x87, 0x28, 0x88, 0xe8, 0xed, 0x27, \ - 0x74, 0x2b, 0x50, 0xfc, 0xf8, 0x02, 0x06, 0xa1, 0xc7, 0x4c, 0xff, 0x2e, \ - 0xf0, 0x5d, 0x57, 0xda, 0x2b, 0xdb, 0xeb, 0x4c, 0xa7, 0x1d, 0x5e, 0x9b, \ - 0x68, 0xc2, 0xcf, 0x23, 0x82, 0x9f, 0xd6, 0x4e, 0x0c, 0x38, 0xdb, 0x4b, \ - 0x49, 0xff, 0xfe, 0xfb, 0x5a, 0xa1, 0xd0, 0x67, 0x72, 0x71, 0xdd, 0xc0, \ - 0xf0, 0x80, 0xc0, 0x7f, 0x2d, 0x18, 0x9c, 0x2d, 0x45, 0xdc, 0xe3, 0xca, \ - 0xbd, 0x63, 0xbf, 0x71, 0x91, 0xa2, 0xb4, 0xaa, 0x13, 0x6c, 0x4a, 0x4f, \ - 0xe3, 0x2e, 0x11, 0x9e, 0x0d, 0xec, 0x14, 0x20, 0xe7, 0x75, 0x10, 0x96, \ - 0x21, 0x12, 0x03, 0x38, 0x41, 0x16, 0xfc, 0x7d, 0xe5, 0x09, 0xc7, 0xcb, \ - 0xee, 0xc3, 0x90, 0xa2, 0x37, 0x4b, 0x97, 0xce, 0x5a, 0x8c, 0x59, 0x21, \ - 0x62, 0xf7, 0x03, 0xaa, 0x39, 0xa7, 0x4d, 0xcb, 0x29, 0x52, 0x7b, 0xaa, \ - 0xf5, 0x8d, 0x34, 0x0b, 0xba, 0x68, 0xad, 0x72, 0x6d, 0x85, 0x9c, 0x33, \ - 0xd5, 0x2d, 0x7c, 0xa9, 0x58, 0xdc, 0x14, 0xd9, 0x34, 0xcc, 0x48, 0x55, \ - 0x31, 0x46, 0xee, 0xa5, 0x2a, 0xdd, 0x5d, 0x59, 0x8a, 0xb4, 0xe2, 0x56, \ - 0x31, 0x66, 0x63, 0x81, 0xc0, 0x3e, 0x39, 0x7c, 0xdc, 0x9d, 0xbd, 0xa1, \ - 0x48, 0xa5, 0x9b, 0x86, 0x3c, 0xa4, 0x1f, 0xfb, 0xba, 0xfc, 0xfa, 0xa4, \ - 0xb8, 0x73, 0xe4, 0x0f, 0xbd, 0xb7, 0xaf, 0x05, 0xdb, 0x98, 0x77, 0x71, \ - 0x45, 0x54, 0xbd, 0x09, 0x6e, 0x7b, 0x90, 0x39, 0x27, 0xe5, 0xec, 0xc9, \ - 0xe6, 0x91, 0x3a, 0x49, 0xa9, 0xb8, 0xa8, 0x5b, 0x02, 0x34, 0x52, 0x4d, \ - 0x05, 0x5f, 0x45, 0x45, 0xfd, 0xd9, 0x04, 0xc1, 0x62 \ + 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ + 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ + 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ + 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ + 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ + 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ + 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ + 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ + 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ + 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ + 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ + 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ + 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ + 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ + 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ + 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ + 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ + 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ + 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ + 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ + 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ + 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ + 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ + 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ + 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ + 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ + 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ + 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ + 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ + 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x01, 0x00, 0x38, 0xa8, 0x54, 0x82, 0xb6, 0x1d, 0xaa, \ + 0xdb, 0x6b, 0x89, 0x21, 0xd1, 0x38, 0x28, 0x61, 0xc9, 0xb0, 0x98, 0xd5, \ + 0x11, 0xfc, 0x36, 0xff, 0x19, 0xfe, 0x32, 0x44, 0xef, 0x08, 0xc7, 0xf1, \ + 0x56, 0xbb, 0xe6, 0x46, 0xfa, 0x82, 0xb6, 0x31, 0x38, 0xb4, 0xa0, 0xe5, \ + 0xa0, 0xae, 0x0c, 0xc4, 0x53, 0x9e, 0x93, 0x4e, 0xe8, 0x0d, 0x9c, 0x2f, \ + 0xb4, 0x04, 0xfd, 0x8a, 0x39, 0xf5, 0x84, 0x77, 0xed, 0x4c, 0xd4, 0xbb, \ + 0x44, 0x7f, 0x73, 0x77, 0xf7, 0xf1, 0x36, 0x97, 0xdc, 0x1e, 0x73, 0x19, \ + 0x5d, 0x50, 0xb8, 0xc3, 0x80, 0xcd, 0x03, 0x57, 0xd2, 0x00, 0xdb, 0x56, \ + 0xe6, 0xc6, 0x35, 0x24, 0x1e, 0x49, 0x7b, 0xd8, 0xd2, 0x72, 0xbb, 0x0b, \ + 0x49, 0x2f, 0xa6, 0x02, 0x3c, 0xaf, 0xd0, 0xec, 0x37, 0x1d, 0xbd, 0x81, \ + 0x8b, 0x1f, 0x30, 0xbb, 0xbc, 0x4d, 0x36, 0xb5, 0x79, 0x7c, 0x87, 0xfb, \ + 0x51, 0xb9, 0xbe, 0xc2, 0xde, 0x92, 0xa8, 0x40, 0x71, 0xbb, 0x72, 0x9b, \ + 0xf8, 0x47, 0xce, 0x6c, 0x04, 0xf8, 0x86, 0xe7, 0xf7, 0x73, 0x3c, 0xe7, \ + 0x84, 0x7d, 0xc2, 0xd7, 0xb7, 0x9d, 0xe8, 0xd4, 0x9b, 0x5f, 0x0a, 0x17, \ + 0x7d, 0xbc, 0xbb, 0xb2, 0xd5, 0x94, 0x0d, 0xe4, 0x49, 0xbf, 0x4f, 0x11, \ + 0x68, 0x53, 0xb2, 0x91, 0xff, 0xc0, 0x69, 0xee, 0xdb, 0x63, 0x93, 0xcb, \ + 0xc9, 0x35, 0x6b, 0x90, 0x09, 0xe2, 0x90, 0xc9, 0xed, 0x27, 0xd6, 0x08, \ + 0xfa, 0x13, 0x4d, 0x62, 0xdd, 0xe2, 0x9e, 0xaa, 0xb5, 0xd4, 0x0e, 0x5c, \ + 0x37, 0x4f, 0xab, 0x55, 0x3b, 0x2d, 0xf1, 0x42, 0x82, 0xc7, 0x34, 0x38, \ + 0x1a, 0x9b, 0xeb, 0xa1, 0x2c, 0x0f, 0x29, 0x31, 0x64, 0x6c, 0xcc, 0x38, \ + 0xfd, 0xa9, 0xd3, 0xd5, 0xd5, 0x71, 0xaf, 0xf0, 0x6d, 0xc0, 0x97, 0xe2, \ + 0x11, 0x2a, 0x0a, 0xdf, 0xfe, 0x02, 0x79, 0x74, 0x75 \ } /* END FILE */ @@ -262,7 +254,7 @@ "-----BEGIN CERTIFICATE-----\r\n" \ "MIIDQTCCAimgAwIBAgIBAzANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" \ "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ - "MTEwMjEyMTQ0NDAwWhcNMjEwMjEyMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "MTkwMjEwMTQ0NDAwWhcNMjkwMjEwMTQ0NDAwWjA7MQswCQYDVQQGEwJOTDERMA8G\r\n" \ "A1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwggEiMA0G\r\n" \ "CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA3zf8F7vglp0/ht6WMn1EpRagzSHx\r\n" \ "mdTs6st8GFgIlKXsm8WL3xoemTiZhx57wI053zhdcHgH057Zk+i5clHFzqMwUqny\r\n" \ @@ -272,88 +264,88 @@ "KNF+AksjoBXyOGVkCeoMbo4bF6BxyLObyavpw/LPh5aPgAIynplYb6LVAgMBAAGj\r\n" \ "UDBOMAwGA1UdEwQFMAMBAf8wHQYDVR0OBBYEFLRa5KWz3tJS9rnVppUP6z68x/3/\r\n" \ "MB8GA1UdIwQYMBaAFLRa5KWz3tJS9rnVppUP6z68x/3/MA0GCSqGSIb3DQEBBQUA\r\n" \ - "A4IBAQABE3OEPfEd/bcJW5ZdU3/VgPNS4tMzh8gnJP/V2FcvFtGylMpQq6YnEBYI\r\n" \ - "yBHAL4DRvlMY5rnXGBp3ODR8MpqHC6AquRTCLzjS57iYff//4QFQqW9n92zctspv\r\n" \ - "czkaPKgjqo1No3Uq0Xaz10rcxyTUPrf5wNVRZ2V0KvllvAAVSzbI4mpdUXztjhST\r\n" \ - "S5A2BeWQAAOr0zq1F7TSRVJpJs7jmB2ai/igkh1IAjcuwV6VwlP+sbw0gjQ0NpGM\r\n" \ - "iHpnlzRAi/tIbtOvMIGOBU2TIfax/5jq1agUx5aPmT5TWAiJPOOP6l5xXnDwxeYS\r\n" \ - "NWqiX9GyusBZjezaCaHabjDLU0qQ\r\n" \ + "A4IBAQB0ZiNRFdia6kskaPnhrqejIRq8YMEGAf2oIPnyZ78xoyERgc35lHGyMtsL\r\n" \ + "hWicNjP4d/hS9As4j5KA2gdNGi5ETA1X7SowWOGsryivSpMSHVy1+HdfWlsYQOzm\r\n" \ + "8o+faQNUm8XzPVmttfAVspxeHSxJZ36Oo+QWZ5wZlCIEyjEdLUId+Tm4Bz3B5jRD\r\n" \ + "zZa/SaqDokq66N2zpbgKKAl3GU2O++fBqP2dSkdQykmTxhLLWRN8FJqhYATyQntZ\r\n" \ + "0QSi3W9HfSZPnFTcPIXeoiPd2pLlxt1hZu8dws2LTXE63uP6MM4LHvWxiuJaWkP/\r\n" \ + "mtxyUALj2pQxRitopORFQdn7AOY5\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ /* This is taken from tests/data_files/test-ca-sha1.crt.der. */ /* BEGIN FILE binary macro TEST_CA_CRT_RSA_SHA1_DER tests/data_files/test-ca-sha1.crt.der */ #define TEST_CA_CRT_RSA_SHA1_DER { \ - 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ - 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ - 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ - 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ - 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ - 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ - 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ - 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ - 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ - 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ - 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ - 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ - 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ - 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ - 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ - 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ - 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ - 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ - 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ - 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ - 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ - 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ - 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ - 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ - 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ - 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ - 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ - 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ - 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ - 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ - 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ - 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ - 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ - 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ - 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ - 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ - 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ - 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, \ - 0x03, 0x82, 0x01, 0x01, 0x00, 0x01, 0x13, 0x73, 0x84, 0x3d, 0xf1, 0x1d, \ - 0xfd, 0xb7, 0x09, 0x5b, 0x96, 0x5d, 0x53, 0x7f, 0xd5, 0x80, 0xf3, 0x52, \ - 0xe2, 0xd3, 0x33, 0x87, 0xc8, 0x27, 0x24, 0xff, 0xd5, 0xd8, 0x57, 0x2f, \ - 0x16, 0xd1, 0xb2, 0x94, 0xca, 0x50, 0xab, 0xa6, 0x27, 0x10, 0x16, 0x08, \ - 0xc8, 0x11, 0xc0, 0x2f, 0x80, 0xd1, 0xbe, 0x53, 0x18, 0xe6, 0xb9, 0xd7, \ - 0x18, 0x1a, 0x77, 0x38, 0x34, 0x7c, 0x32, 0x9a, 0x87, 0x0b, 0xa0, 0x2a, \ - 0xb9, 0x14, 0xc2, 0x2f, 0x38, 0xd2, 0xe7, 0xb8, 0x98, 0x7d, 0xff, 0xff, \ - 0xe1, 0x01, 0x50, 0xa9, 0x6f, 0x67, 0xf7, 0x6c, 0xdc, 0xb6, 0xca, 0x6f, \ - 0x73, 0x39, 0x1a, 0x3c, 0xa8, 0x23, 0xaa, 0x8d, 0x4d, 0xa3, 0x75, 0x2a, \ - 0xd1, 0x76, 0xb3, 0xd7, 0x4a, 0xdc, 0xc7, 0x24, 0xd4, 0x3e, 0xb7, 0xf9, \ - 0xc0, 0xd5, 0x51, 0x67, 0x65, 0x74, 0x2a, 0xf9, 0x65, 0xbc, 0x00, 0x15, \ - 0x4b, 0x36, 0xc8, 0xe2, 0x6a, 0x5d, 0x51, 0x7c, 0xed, 0x8e, 0x14, 0x93, \ - 0x4b, 0x90, 0x36, 0x05, 0xe5, 0x90, 0x00, 0x03, 0xab, 0xd3, 0x3a, 0xb5, \ - 0x17, 0xb4, 0xd2, 0x45, 0x52, 0x69, 0x26, 0xce, 0xe3, 0x98, 0x1d, 0x9a, \ - 0x8b, 0xf8, 0xa0, 0x92, 0x1d, 0x48, 0x02, 0x37, 0x2e, 0xc1, 0x5e, 0x95, \ - 0xc2, 0x53, 0xfe, 0xb1, 0xbc, 0x34, 0x82, 0x34, 0x34, 0x36, 0x91, 0x8c, \ - 0x88, 0x7a, 0x67, 0x97, 0x34, 0x40, 0x8b, 0xfb, 0x48, 0x6e, 0xd3, 0xaf, \ - 0x30, 0x81, 0x8e, 0x05, 0x4d, 0x93, 0x21, 0xf6, 0xb1, 0xff, 0x98, 0xea, \ - 0xd5, 0xa8, 0x14, 0xc7, 0x96, 0x8f, 0x99, 0x3e, 0x53, 0x58, 0x08, 0x89, \ - 0x3c, 0xe3, 0x8f, 0xea, 0x5e, 0x71, 0x5e, 0x70, 0xf0, 0xc5, 0xe6, 0x12, \ - 0x35, 0x6a, 0xa2, 0x5f, 0xd1, 0xb2, 0xba, 0xc0, 0x59, 0x8d, 0xec, 0xda, \ - 0x09, 0xa1, 0xda, 0x6e, 0x30, 0xcb, 0x53, 0x4a, 0x90 \ + 0x30, 0x82, 0x03, 0x41, 0x30, 0x82, 0x02, 0x29, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x03, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x30, \ + 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x30, 0x5a, 0x30, 0x3b, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ + 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, 0x01, \ + 0x01, 0x00, 0xc0, 0xdf, 0x37, 0xfc, 0x17, 0xbb, 0xe0, 0x96, 0x9d, 0x3f, \ + 0x86, 0xde, 0x96, 0x32, 0x7d, 0x44, 0xa5, 0x16, 0xa0, 0xcd, 0x21, 0xf1, \ + 0x99, 0xd4, 0xec, 0xea, 0xcb, 0x7c, 0x18, 0x58, 0x08, 0x94, 0xa5, 0xec, \ + 0x9b, 0xc5, 0x8b, 0xdf, 0x1a, 0x1e, 0x99, 0x38, 0x99, 0x87, 0x1e, 0x7b, \ + 0xc0, 0x8d, 0x39, 0xdf, 0x38, 0x5d, 0x70, 0x78, 0x07, 0xd3, 0x9e, 0xd9, \ + 0x93, 0xe8, 0xb9, 0x72, 0x51, 0xc5, 0xce, 0xa3, 0x30, 0x52, 0xa9, 0xf2, \ + 0xe7, 0x40, 0x70, 0x14, 0xcb, 0x44, 0xa2, 0x72, 0x0b, 0xc2, 0xe5, 0x40, \ + 0xf9, 0x3e, 0xe5, 0xa6, 0x0e, 0xb3, 0xf9, 0xec, 0x4a, 0x63, 0xc0, 0xb8, \ + 0x29, 0x00, 0x74, 0x9c, 0x57, 0x3b, 0xa8, 0xa5, 0x04, 0x90, 0x71, 0xf1, \ + 0xbd, 0x83, 0xd9, 0x3f, 0xd6, 0xa5, 0xe2, 0x3c, 0x2a, 0x8f, 0xef, 0x27, \ + 0x60, 0xc3, 0xc6, 0x9f, 0xcb, 0xba, 0xec, 0x60, 0x7d, 0xb7, 0xe6, 0x84, \ + 0x32, 0xbe, 0x4f, 0xfb, 0x58, 0x26, 0x22, 0x03, 0x5b, 0xd4, 0xb4, 0xd5, \ + 0xfb, 0xf5, 0xe3, 0x96, 0x2e, 0x70, 0xc0, 0xe4, 0x2e, 0xbd, 0xfc, 0x2e, \ + 0xee, 0xe2, 0x41, 0x55, 0xc0, 0x34, 0x2e, 0x7d, 0x24, 0x72, 0x69, 0xcb, \ + 0x47, 0xb1, 0x14, 0x40, 0x83, 0x7d, 0x67, 0xf4, 0x86, 0xf6, 0x31, 0xab, \ + 0xf1, 0x79, 0xa4, 0xb2, 0xb5, 0x2e, 0x12, 0xf9, 0x84, 0x17, 0xf0, 0x62, \ + 0x6f, 0x27, 0x3e, 0x13, 0x58, 0xb1, 0x54, 0x0d, 0x21, 0x9a, 0x73, 0x37, \ + 0xa1, 0x30, 0xcf, 0x6f, 0x92, 0xdc, 0xf6, 0xe9, 0xfc, 0xac, 0xdb, 0x2e, \ + 0x28, 0xd1, 0x7e, 0x02, 0x4b, 0x23, 0xa0, 0x15, 0xf2, 0x38, 0x65, 0x64, \ + 0x09, 0xea, 0x0c, 0x6e, 0x8e, 0x1b, 0x17, 0xa0, 0x71, 0xc8, 0xb3, 0x9b, \ + 0xc9, 0xab, 0xe9, 0xc3, 0xf2, 0xcf, 0x87, 0x96, 0x8f, 0x80, 0x02, 0x32, \ + 0x9e, 0x99, 0x58, 0x6f, 0xa2, 0xd5, 0x02, 0x03, 0x01, 0x00, 0x01, 0xa3, \ + 0x50, 0x30, 0x4e, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x05, \ + 0x30, 0x03, 0x01, 0x01, 0xff, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, \ + 0x04, 0x16, 0x04, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, \ + 0xf6, 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, \ + 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, \ + 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, \ + 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, \ + 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, \ + 0x03, 0x82, 0x01, 0x01, 0x00, 0x74, 0x66, 0x23, 0x51, 0x15, 0xd8, 0x9a, \ + 0xea, 0x4b, 0x24, 0x68, 0xf9, 0xe1, 0xae, 0xa7, 0xa3, 0x21, 0x1a, 0xbc, \ + 0x60, 0xc1, 0x06, 0x01, 0xfd, 0xa8, 0x20, 0xf9, 0xf2, 0x67, 0xbf, 0x31, \ + 0xa3, 0x21, 0x11, 0x81, 0xcd, 0xf9, 0x94, 0x71, 0xb2, 0x32, 0xdb, 0x0b, \ + 0x85, 0x68, 0x9c, 0x36, 0x33, 0xf8, 0x77, 0xf8, 0x52, 0xf4, 0x0b, 0x38, \ + 0x8f, 0x92, 0x80, 0xda, 0x07, 0x4d, 0x1a, 0x2e, 0x44, 0x4c, 0x0d, 0x57, \ + 0xed, 0x2a, 0x30, 0x58, 0xe1, 0xac, 0xaf, 0x28, 0xaf, 0x4a, 0x93, 0x12, \ + 0x1d, 0x5c, 0xb5, 0xf8, 0x77, 0x5f, 0x5a, 0x5b, 0x18, 0x40, 0xec, 0xe6, \ + 0xf2, 0x8f, 0x9f, 0x69, 0x03, 0x54, 0x9b, 0xc5, 0xf3, 0x3d, 0x59, 0xad, \ + 0xb5, 0xf0, 0x15, 0xb2, 0x9c, 0x5e, 0x1d, 0x2c, 0x49, 0x67, 0x7e, 0x8e, \ + 0xa3, 0xe4, 0x16, 0x67, 0x9c, 0x19, 0x94, 0x22, 0x04, 0xca, 0x31, 0x1d, \ + 0x2d, 0x42, 0x1d, 0xf9, 0x39, 0xb8, 0x07, 0x3d, 0xc1, 0xe6, 0x34, 0x43, \ + 0xcd, 0x96, 0xbf, 0x49, 0xaa, 0x83, 0xa2, 0x4a, 0xba, 0xe8, 0xdd, 0xb3, \ + 0xa5, 0xb8, 0x0a, 0x28, 0x09, 0x77, 0x19, 0x4d, 0x8e, 0xfb, 0xe7, 0xc1, \ + 0xa8, 0xfd, 0x9d, 0x4a, 0x47, 0x50, 0xca, 0x49, 0x93, 0xc6, 0x12, 0xcb, \ + 0x59, 0x13, 0x7c, 0x14, 0x9a, 0xa1, 0x60, 0x04, 0xf2, 0x42, 0x7b, 0x59, \ + 0xd1, 0x04, 0xa2, 0xdd, 0x6f, 0x47, 0x7d, 0x26, 0x4f, 0x9c, 0x54, 0xdc, \ + 0x3c, 0x85, 0xde, 0xa2, 0x23, 0xdd, 0xda, 0x92, 0xe5, 0xc6, 0xdd, 0x61, \ + 0x66, 0xef, 0x1d, 0xc2, 0xcd, 0x8b, 0x4d, 0x71, 0x3a, 0xde, 0xe3, 0xfa, \ + 0x30, 0xce, 0x0b, 0x1e, 0xf5, 0xb1, 0x8a, 0xe2, 0x5a, 0x5a, 0x43, 0xff, \ + 0x9a, 0xdc, 0x72, 0x50, 0x02, 0xe3, 0xda, 0x94, 0x31, 0x46, 0x2b, 0x68, \ + 0xa4, 0xe4, 0x45, 0x41, 0xd9, 0xfb, 0x00, 0xe6, 0x39 \ } /* END FILE */ @@ -617,7 +609,7 @@ "-----BEGIN CERTIFICATE-----\r\n" \ "MIIDNzCCAh+gAwIBAgIBAjANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ - "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "MTkwMjEwMTQ0NDA2WhcNMjkwMjEwMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ "A1UECgwIUG9sYXJTU0wxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN\r\n" \ "AQEBBQADggEPADCCAQoCggEBAMFNo93nzR3RBNdJcriZrA545Do8Ss86ExbQWuTN\r\n" \ "owCIp+4ea5anUrSQ7y1yej4kmvy2NKwk9XfgJmSMnLAofaHa6ozmyRyWvP7BBFKz\r\n" \ @@ -626,88 +618,88 @@ "hYvai0Re4hjGYi/HZo36Xdh98yeJKQHFkA4/J/EwyEoO79bex8cna8cFPXrEAjya\r\n" \ "HT4P6DSYW8tzS1KW2BGiLICIaTla0w+w3lkvEcf36hIBMJcCAwEAAaNNMEswCQYD\r\n" \ "VR0TBAIwADAdBgNVHQ4EFgQUpQXoZLjc32APUBJNYKhkr02LQ5MwHwYDVR0jBBgw\r\n" \ - "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQELBQADggEBAGGEshT5\r\n" \ - "kvnRmLVScVeUEdwIrvW7ezbGbUvJ8VxeJ79/HSjlLiGbMc4uUathwtzEdi9R/4C5\r\n" \ - "DXBNeEPTkbB+fhG1W06iHYj/Dp8+aaG7fuDxKVKHVZSqBnmQLn73ymyclZNHii5A\r\n" \ - "3nTS8WUaHAzxN/rajOtoM7aH1P9tULpHrl+7HOeLMpxUnwI12ZqZaLIzxbcdJVcr\r\n" \ - "ra2F00aXCGkYVLvyvbZIq7LC+yVysej5gCeQYD7VFOEks0jhFjrS06gP0/XnWv6v\r\n" \ - "eBoPez9d+CCjkrhseiWzXOiriIMICX48EloO/DrsMRAtvlwq7EDz4QhILz6ffndm\r\n" \ - "e4K1cVANRPN2o9Y=\r\n" \ + "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQELBQADggEBAC465FJh\r\n" \ + "Pqel7zJngHIHJrqj/wVAxGAFOTF396XKATGAp+HRCqJ81Ry60CNK1jDzk8dv6M6U\r\n" \ + "HoS7RIFiM/9rXQCbJfiPD5xMTejZp5n5UYHAmxsxDaazfA5FuBhkfokKK6jD4Eq9\r\n" \ + "1C94xGKb6X4/VkaPF7cqoBBw/bHxawXc0UEPjqayiBpCYU/rJoVZgLqFVP7Px3sv\r\n" \ + "a1nOrNx8rPPI1hJ+ZOg8maiPTxHZnBVLakSSLQy/sWeWyazO1RnrbxjrbgQtYKz0\r\n" \ + "e3nwGpu1w13vfckFmUSBhHXH7AAS/HpKC4IH7G2GAk3+n8iSSN71sZzpxonQwVbo\r\n" \ + "pMZqLmbBm/7WPLc=\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ /* This is taken from tests/data_files/server2-sha256.crt.der. */ /* BEGIN FILE binary macro TEST_SRV_CRT_RSA_SHA256_DER tests/data_files/server2-sha256.crt.der */ #define TEST_SRV_CRT_RSA_SHA256_DER { \ - 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ - 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ - 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ - 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ - 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ - 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ - 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ - 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ - 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ - 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ - 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ - 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ - 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ - 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ - 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ - 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ - 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ - 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ - 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ - 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ - 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ - 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ - 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ - 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ - 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ - 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ - 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ - 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ - 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ - 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ - 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ - 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ - 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ - 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ - 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ - 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, \ - 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x61, 0x84, 0xb2, 0x14, 0xf9, \ - 0x92, 0xf9, 0xd1, 0x98, 0xb5, 0x52, 0x71, 0x57, 0x94, 0x11, 0xdc, 0x08, \ - 0xae, 0xf5, 0xbb, 0x7b, 0x36, 0xc6, 0x6d, 0x4b, 0xc9, 0xf1, 0x5c, 0x5e, \ - 0x27, 0xbf, 0x7f, 0x1d, 0x28, 0xe5, 0x2e, 0x21, 0x9b, 0x31, 0xce, 0x2e, \ - 0x51, 0xab, 0x61, 0xc2, 0xdc, 0xc4, 0x76, 0x2f, 0x51, 0xff, 0x80, 0xb9, \ - 0x0d, 0x70, 0x4d, 0x78, 0x43, 0xd3, 0x91, 0xb0, 0x7e, 0x7e, 0x11, 0xb5, \ - 0x5b, 0x4e, 0xa2, 0x1d, 0x88, 0xff, 0x0e, 0x9f, 0x3e, 0x69, 0xa1, 0xbb, \ - 0x7e, 0xe0, 0xf1, 0x29, 0x52, 0x87, 0x55, 0x94, 0xaa, 0x06, 0x79, 0x90, \ - 0x2e, 0x7e, 0xf7, 0xca, 0x6c, 0x9c, 0x95, 0x93, 0x47, 0x8a, 0x2e, 0x40, \ - 0xde, 0x74, 0xd2, 0xf1, 0x65, 0x1a, 0x1c, 0x0c, 0xf1, 0x37, 0xfa, 0xda, \ - 0x8c, 0xeb, 0x68, 0x33, 0xb6, 0x87, 0xd4, 0xff, 0x6d, 0x50, 0xba, 0x47, \ - 0xae, 0x5f, 0xbb, 0x1c, 0xe7, 0x8b, 0x32, 0x9c, 0x54, 0x9f, 0x02, 0x35, \ - 0xd9, 0x9a, 0x99, 0x68, 0xb2, 0x33, 0xc5, 0xb7, 0x1d, 0x25, 0x57, 0x2b, \ - 0xad, 0xad, 0x85, 0xd3, 0x46, 0x97, 0x08, 0x69, 0x18, 0x54, 0xbb, 0xf2, \ - 0xbd, 0xb6, 0x48, 0xab, 0xb2, 0xc2, 0xfb, 0x25, 0x72, 0xb1, 0xe8, 0xf9, \ - 0x80, 0x27, 0x90, 0x60, 0x3e, 0xd5, 0x14, 0xe1, 0x24, 0xb3, 0x48, 0xe1, \ - 0x16, 0x3a, 0xd2, 0xd3, 0xa8, 0x0f, 0xd3, 0xf5, 0xe7, 0x5a, 0xfe, 0xaf, \ - 0x78, 0x1a, 0x0f, 0x7b, 0x3f, 0x5d, 0xf8, 0x20, 0xa3, 0x92, 0xb8, 0x6c, \ - 0x7a, 0x25, 0xb3, 0x5c, 0xe8, 0xab, 0x88, 0x83, 0x08, 0x09, 0x7e, 0x3c, \ - 0x12, 0x5a, 0x0e, 0xfc, 0x3a, 0xec, 0x31, 0x10, 0x2d, 0xbe, 0x5c, 0x2a, \ - 0xec, 0x40, 0xf3, 0xe1, 0x08, 0x48, 0x2f, 0x3e, 0x9f, 0x7e, 0x77, 0x66, \ - 0x7b, 0x82, 0xb5, 0x71, 0x50, 0x0d, 0x44, 0xf3, 0x76, 0xa3, 0xd6 \ + 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ + 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ + 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ + 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ + 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ + 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ + 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ + 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ + 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ + 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ + 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ + 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ + 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ + 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ + 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ + 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ + 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ + 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ + 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ + 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ + 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ + 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ + 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ + 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ + 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ + 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ + 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ + 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ + 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, \ + 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x2e, 0x3a, 0xe4, 0x52, 0x61, \ + 0x3e, 0xa7, 0xa5, 0xef, 0x32, 0x67, 0x80, 0x72, 0x07, 0x26, 0xba, 0xa3, \ + 0xff, 0x05, 0x40, 0xc4, 0x60, 0x05, 0x39, 0x31, 0x77, 0xf7, 0xa5, 0xca, \ + 0x01, 0x31, 0x80, 0xa7, 0xe1, 0xd1, 0x0a, 0xa2, 0x7c, 0xd5, 0x1c, 0xba, \ + 0xd0, 0x23, 0x4a, 0xd6, 0x30, 0xf3, 0x93, 0xc7, 0x6f, 0xe8, 0xce, 0x94, \ + 0x1e, 0x84, 0xbb, 0x44, 0x81, 0x62, 0x33, 0xff, 0x6b, 0x5d, 0x00, 0x9b, \ + 0x25, 0xf8, 0x8f, 0x0f, 0x9c, 0x4c, 0x4d, 0xe8, 0xd9, 0xa7, 0x99, 0xf9, \ + 0x51, 0x81, 0xc0, 0x9b, 0x1b, 0x31, 0x0d, 0xa6, 0xb3, 0x7c, 0x0e, 0x45, \ + 0xb8, 0x18, 0x64, 0x7e, 0x89, 0x0a, 0x2b, 0xa8, 0xc3, 0xe0, 0x4a, 0xbd, \ + 0xd4, 0x2f, 0x78, 0xc4, 0x62, 0x9b, 0xe9, 0x7e, 0x3f, 0x56, 0x46, 0x8f, \ + 0x17, 0xb7, 0x2a, 0xa0, 0x10, 0x70, 0xfd, 0xb1, 0xf1, 0x6b, 0x05, 0xdc, \ + 0xd1, 0x41, 0x0f, 0x8e, 0xa6, 0xb2, 0x88, 0x1a, 0x42, 0x61, 0x4f, 0xeb, \ + 0x26, 0x85, 0x59, 0x80, 0xba, 0x85, 0x54, 0xfe, 0xcf, 0xc7, 0x7b, 0x2f, \ + 0x6b, 0x59, 0xce, 0xac, 0xdc, 0x7c, 0xac, 0xf3, 0xc8, 0xd6, 0x12, 0x7e, \ + 0x64, 0xe8, 0x3c, 0x99, 0xa8, 0x8f, 0x4f, 0x11, 0xd9, 0x9c, 0x15, 0x4b, \ + 0x6a, 0x44, 0x92, 0x2d, 0x0c, 0xbf, 0xb1, 0x67, 0x96, 0xc9, 0xac, 0xce, \ + 0xd5, 0x19, 0xeb, 0x6f, 0x18, 0xeb, 0x6e, 0x04, 0x2d, 0x60, 0xac, 0xf4, \ + 0x7b, 0x79, 0xf0, 0x1a, 0x9b, 0xb5, 0xc3, 0x5d, 0xef, 0x7d, 0xc9, 0x05, \ + 0x99, 0x44, 0x81, 0x84, 0x75, 0xc7, 0xec, 0x00, 0x12, 0xfc, 0x7a, 0x4a, \ + 0x0b, 0x82, 0x07, 0xec, 0x6d, 0x86, 0x02, 0x4d, 0xfe, 0x9f, 0xc8, 0x92, \ + 0x48, 0xde, 0xf5, 0xb1, 0x9c, 0xe9, 0xc6, 0x89, 0xd0, 0xc1, 0x56, 0xe8, \ + 0xa4, 0xc6, 0x6a, 0x2e, 0x66, 0xc1, 0x9b, 0xfe, 0xd6, 0x3c, 0xb7 \ } /* END FILE */ @@ -717,7 +709,7 @@ "-----BEGIN CERTIFICATE-----\r\n" \ "MIIDNzCCAh+gAwIBAgIBAjANBgkqhkiG9w0BAQUFADA7MQswCQYDVQQGEwJOTDER\r\n" \ "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ - "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "MTkwMjEwMTQ0NDA2WhcNMjkwMjEwMTQ0NDA2WjA0MQswCQYDVQQGEwJOTDERMA8G\r\n" \ "A1UECgwIUG9sYXJTU0wxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN\r\n" \ "AQEBBQADggEPADCCAQoCggEBAMFNo93nzR3RBNdJcriZrA545Do8Ss86ExbQWuTN\r\n" \ "owCIp+4ea5anUrSQ7y1yej4kmvy2NKwk9XfgJmSMnLAofaHa6ozmyRyWvP7BBFKz\r\n" \ @@ -726,88 +718,88 @@ "hYvai0Re4hjGYi/HZo36Xdh98yeJKQHFkA4/J/EwyEoO79bex8cna8cFPXrEAjya\r\n" \ "HT4P6DSYW8tzS1KW2BGiLICIaTla0w+w3lkvEcf36hIBMJcCAwEAAaNNMEswCQYD\r\n" \ "VR0TBAIwADAdBgNVHQ4EFgQUpQXoZLjc32APUBJNYKhkr02LQ5MwHwYDVR0jBBgw\r\n" \ - "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQEFBQADggEBAAFzC0rF\r\n" \ - "y6De8WMcdgQrEw3AhBHFjzqnxZw1ene4IBSC7lTw8rBSy3jOWQdPUWn+0y/pCeeF\r\n" \ - "kti6sevFdl1hLemGtd4q+T9TKEKGg3ND4ARfB5AUZZ9uEHq8WBkiwus5clGS17Qd\r\n" \ - "dS/TOisB59tQruLx1E1bPLtBKyqk4koC5WAULJwfpswGSyWJTpYwIpxcWE3D2tBu\r\n" \ - "UB6MZfXZFzWmWEOyKbeoXjXe8GBCGgHLywvYDsGQ36HSGtEsAvR2QaTLSxWYcfk1\r\n" \ - "fbDn4jSWkb4yZy1r01UEigFQtONieGwRFaUqEcFJHJvEEGVgh9keaVlOj2vrwf5r\r\n" \ - "4mN4lW7gLdenN6g=\r\n" \ + "FoAUtFrkpbPe0lL2udWmlQ/rPrzH/f8wDQYJKoZIhvcNAQEFBQADggEBAJklg3Q4\r\n" \ + "cB7v7BzsxM/vLyKccO6op0/gZzM4ghuLq2Y32kl0sM6kSNUUmduuq3u/+GmUZN2A\r\n" \ + "O/7c+Hw7hDFEIvZk98aBGjCLqn3DmgHIv8ToQ67nellQxx2Uj309PdgjNi/r9HOc\r\n" \ + "KNAYPbBcg6MJGWWj2TI6vNaceios/DhOYx5V0j5nfqSJ/pnU0g9Ign2LAhgYpGJE\r\n" \ + "iEM9wW7hEMkwmk0h/sqZsrJsGH5YsF/VThSq/JVO1e2mZH2vruyZKJVBq+8tDNYp\r\n" \ + "HkK6tSyVYQhzIt3StMJWKMl/o5k2AYz6tSC164+1oG+ML3LWg8XrGKa91H4UOKap\r\n" \ + "Awgk0+4m0T25cNs=\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ /* This is taken from tests/data_files/server2.crt.der. */ /* BEGIN FILE binary macro TEST_SRV_CRT_RSA_SHA1_DER tests/data_files/server2.crt.der */ #define TEST_SRV_CRT_RSA_SHA1_DER { \ - 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ - 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ - 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ - 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ - 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ - 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ - 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ - 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ - 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ - 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ - 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ - 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ - 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ - 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ - 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ - 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ - 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ - 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ - 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ - 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ - 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ - 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ - 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ - 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ - 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ - 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ - 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ - 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ - 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ - 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ - 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ - 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ - 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ - 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ - 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ - 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, \ - 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x01, 0x73, 0x0b, 0x4a, 0xc5, \ - 0xcb, 0xa0, 0xde, 0xf1, 0x63, 0x1c, 0x76, 0x04, 0x2b, 0x13, 0x0d, 0xc0, \ - 0x84, 0x11, 0xc5, 0x8f, 0x3a, 0xa7, 0xc5, 0x9c, 0x35, 0x7a, 0x77, 0xb8, \ - 0x20, 0x14, 0x82, 0xee, 0x54, 0xf0, 0xf2, 0xb0, 0x52, 0xcb, 0x78, 0xce, \ - 0x59, 0x07, 0x4f, 0x51, 0x69, 0xfe, 0xd3, 0x2f, 0xe9, 0x09, 0xe7, 0x85, \ - 0x92, 0xd8, 0xba, 0xb1, 0xeb, 0xc5, 0x76, 0x5d, 0x61, 0x2d, 0xe9, 0x86, \ - 0xb5, 0xde, 0x2a, 0xf9, 0x3f, 0x53, 0x28, 0x42, 0x86, 0x83, 0x73, 0x43, \ - 0xe0, 0x04, 0x5f, 0x07, 0x90, 0x14, 0x65, 0x9f, 0x6e, 0x10, 0x7a, 0xbc, \ - 0x58, 0x19, 0x22, 0xc2, 0xeb, 0x39, 0x72, 0x51, 0x92, 0xd7, 0xb4, 0x1d, \ - 0x75, 0x2f, 0xd3, 0x3a, 0x2b, 0x01, 0xe7, 0xdb, 0x50, 0xae, 0xe2, 0xf1, \ - 0xd4, 0x4d, 0x5b, 0x3c, 0xbb, 0x41, 0x2b, 0x2a, 0xa4, 0xe2, 0x4a, 0x02, \ - 0xe5, 0x60, 0x14, 0x2c, 0x9c, 0x1f, 0xa6, 0xcc, 0x06, 0x4b, 0x25, 0x89, \ - 0x4e, 0x96, 0x30, 0x22, 0x9c, 0x5c, 0x58, 0x4d, 0xc3, 0xda, 0xd0, 0x6e, \ - 0x50, 0x1e, 0x8c, 0x65, 0xf5, 0xd9, 0x17, 0x35, 0xa6, 0x58, 0x43, 0xb2, \ - 0x29, 0xb7, 0xa8, 0x5e, 0x35, 0xde, 0xf0, 0x60, 0x42, 0x1a, 0x01, 0xcb, \ - 0xcb, 0x0b, 0xd8, 0x0e, 0xc1, 0x90, 0xdf, 0xa1, 0xd2, 0x1a, 0xd1, 0x2c, \ - 0x02, 0xf4, 0x76, 0x41, 0xa4, 0xcb, 0x4b, 0x15, 0x98, 0x71, 0xf9, 0x35, \ - 0x7d, 0xb0, 0xe7, 0xe2, 0x34, 0x96, 0x91, 0xbe, 0x32, 0x67, 0x2d, 0x6b, \ - 0xd3, 0x55, 0x04, 0x8a, 0x01, 0x50, 0xb4, 0xe3, 0x62, 0x78, 0x6c, 0x11, \ - 0x15, 0xa5, 0x2a, 0x11, 0xc1, 0x49, 0x1c, 0x9b, 0xc4, 0x10, 0x65, 0x60, \ - 0x87, 0xd9, 0x1e, 0x69, 0x59, 0x4e, 0x8f, 0x6b, 0xeb, 0xc1, 0xfe, 0x6b, \ - 0xe2, 0x63, 0x78, 0x95, 0x6e, 0xe0, 0x2d, 0xd7, 0xa7, 0x37, 0xa8 \ + 0x30, 0x82, 0x03, 0x37, 0x30, 0x82, 0x02, 0x1f, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x02, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x05, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x34, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x12, 0x30, 0x10, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x82, \ + 0x01, 0x22, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, \ + 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, \ + 0x01, 0x0a, 0x02, 0x82, 0x01, 0x01, 0x00, 0xc1, 0x4d, 0xa3, 0xdd, 0xe7, \ + 0xcd, 0x1d, 0xd1, 0x04, 0xd7, 0x49, 0x72, 0xb8, 0x99, 0xac, 0x0e, 0x78, \ + 0xe4, 0x3a, 0x3c, 0x4a, 0xcf, 0x3a, 0x13, 0x16, 0xd0, 0x5a, 0xe4, 0xcd, \ + 0xa3, 0x00, 0x88, 0xa7, 0xee, 0x1e, 0x6b, 0x96, 0xa7, 0x52, 0xb4, 0x90, \ + 0xef, 0x2d, 0x72, 0x7a, 0x3e, 0x24, 0x9a, 0xfc, 0xb6, 0x34, 0xac, 0x24, \ + 0xf5, 0x77, 0xe0, 0x26, 0x64, 0x8c, 0x9c, 0xb0, 0x28, 0x7d, 0xa1, 0xda, \ + 0xea, 0x8c, 0xe6, 0xc9, 0x1c, 0x96, 0xbc, 0xfe, 0xc1, 0x04, 0x52, 0xb3, \ + 0x36, 0xd4, 0xa3, 0xfa, 0xe1, 0xb1, 0x76, 0xd8, 0x90, 0xc1, 0x61, 0xb4, \ + 0x66, 0x52, 0x36, 0xa2, 0x26, 0x53, 0xaa, 0xab, 0x74, 0x5e, 0x07, 0x7d, \ + 0x19, 0x82, 0xdb, 0x2a, 0xd8, 0x1f, 0xa0, 0xd9, 0x0d, 0x1c, 0x2d, 0x49, \ + 0x66, 0xf7, 0x5b, 0x25, 0x73, 0x46, 0xe8, 0x0b, 0x8a, 0x4f, 0x69, 0x0c, \ + 0xb5, 0x00, 0x90, 0xe1, 0xda, 0x82, 0x10, 0x66, 0x7d, 0xae, 0x54, 0x2b, \ + 0x8b, 0x65, 0x79, 0x91, 0xa1, 0xe2, 0x61, 0xc3, 0xcd, 0x40, 0x49, 0x08, \ + 0xee, 0x68, 0x0c, 0xf1, 0x8b, 0x86, 0xd2, 0x46, 0xbf, 0xd0, 0xb8, 0xaa, \ + 0x11, 0x03, 0x1e, 0x7f, 0x56, 0xa8, 0x1a, 0x1e, 0x44, 0x18, 0x0f, 0x0f, \ + 0x85, 0x8b, 0xda, 0x8b, 0x44, 0x5e, 0xe2, 0x18, 0xc6, 0x62, 0x2f, 0xc7, \ + 0x66, 0x8d, 0xfa, 0x5d, 0xd8, 0x7d, 0xf3, 0x27, 0x89, 0x29, 0x01, 0xc5, \ + 0x90, 0x0e, 0x3f, 0x27, 0xf1, 0x30, 0xc8, 0x4a, 0x0e, 0xef, 0xd6, 0xde, \ + 0xc7, 0xc7, 0x27, 0x6b, 0xc7, 0x05, 0x3d, 0x7a, 0xc4, 0x02, 0x3c, 0x9a, \ + 0x1d, 0x3e, 0x0f, 0xe8, 0x34, 0x98, 0x5b, 0xcb, 0x73, 0x4b, 0x52, 0x96, \ + 0xd8, 0x11, 0xa2, 0x2c, 0x80, 0x88, 0x69, 0x39, 0x5a, 0xd3, 0x0f, 0xb0, \ + 0xde, 0x59, 0x2f, 0x11, 0xc7, 0xf7, 0xea, 0x12, 0x01, 0x30, 0x97, 0x02, \ + 0x03, 0x01, 0x00, 0x01, 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, \ + 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0xa5, 0x05, 0xe8, 0x64, 0xb8, 0xdc, \ + 0xdf, 0x60, 0x0f, 0x50, 0x12, 0x4d, 0x60, 0xa8, 0x64, 0xaf, 0x4d, 0x8b, \ + 0x43, 0x93, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, \ + 0x16, 0x80, 0x14, 0xb4, 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, \ + 0xb9, 0xd5, 0xa6, 0x95, 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, \ + 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x05, \ + 0x05, 0x00, 0x03, 0x82, 0x01, 0x01, 0x00, 0x99, 0x25, 0x83, 0x74, 0x38, \ + 0x70, 0x1e, 0xef, 0xec, 0x1c, 0xec, 0xc4, 0xcf, 0xef, 0x2f, 0x22, 0x9c, \ + 0x70, 0xee, 0xa8, 0xa7, 0x4f, 0xe0, 0x67, 0x33, 0x38, 0x82, 0x1b, 0x8b, \ + 0xab, 0x66, 0x37, 0xda, 0x49, 0x74, 0xb0, 0xce, 0xa4, 0x48, 0xd5, 0x14, \ + 0x99, 0xdb, 0xae, 0xab, 0x7b, 0xbf, 0xf8, 0x69, 0x94, 0x64, 0xdd, 0x80, \ + 0x3b, 0xfe, 0xdc, 0xf8, 0x7c, 0x3b, 0x84, 0x31, 0x44, 0x22, 0xf6, 0x64, \ + 0xf7, 0xc6, 0x81, 0x1a, 0x30, 0x8b, 0xaa, 0x7d, 0xc3, 0x9a, 0x01, 0xc8, \ + 0xbf, 0xc4, 0xe8, 0x43, 0xae, 0xe7, 0x7a, 0x59, 0x50, 0xc7, 0x1d, 0x94, \ + 0x8f, 0x7d, 0x3d, 0x3d, 0xd8, 0x23, 0x36, 0x2f, 0xeb, 0xf4, 0x73, 0x9c, \ + 0x28, 0xd0, 0x18, 0x3d, 0xb0, 0x5c, 0x83, 0xa3, 0x09, 0x19, 0x65, 0xa3, \ + 0xd9, 0x32, 0x3a, 0xbc, 0xd6, 0x9c, 0x7a, 0x2a, 0x2c, 0xfc, 0x38, 0x4e, \ + 0x63, 0x1e, 0x55, 0xd2, 0x3e, 0x67, 0x7e, 0xa4, 0x89, 0xfe, 0x99, 0xd4, \ + 0xd2, 0x0f, 0x48, 0x82, 0x7d, 0x8b, 0x02, 0x18, 0x18, 0xa4, 0x62, 0x44, \ + 0x88, 0x43, 0x3d, 0xc1, 0x6e, 0xe1, 0x10, 0xc9, 0x30, 0x9a, 0x4d, 0x21, \ + 0xfe, 0xca, 0x99, 0xb2, 0xb2, 0x6c, 0x18, 0x7e, 0x58, 0xb0, 0x5f, 0xd5, \ + 0x4e, 0x14, 0xaa, 0xfc, 0x95, 0x4e, 0xd5, 0xed, 0xa6, 0x64, 0x7d, 0xaf, \ + 0xae, 0xec, 0x99, 0x28, 0x95, 0x41, 0xab, 0xef, 0x2d, 0x0c, 0xd6, 0x29, \ + 0x1e, 0x42, 0xba, 0xb5, 0x2c, 0x95, 0x61, 0x08, 0x73, 0x22, 0xdd, 0xd2, \ + 0xb4, 0xc2, 0x56, 0x28, 0xc9, 0x7f, 0xa3, 0x99, 0x36, 0x01, 0x8c, 0xfa, \ + 0xb5, 0x20, 0xb5, 0xeb, 0x8f, 0xb5, 0xa0, 0x6f, 0x8c, 0x2f, 0x72, 0xd6, \ + 0x83, 0xc5, 0xeb, 0x18, 0xa6, 0xbd, 0xd4, 0x7e, 0x14, 0x38, 0xa6, 0xa9, \ + 0x03, 0x08, 0x24, 0xd3, 0xee, 0x26, 0xd1, 0x3d, 0xb9, 0x70, 0xdb \ } /* END FILE */ @@ -966,71 +958,64 @@ /* BEGIN FILE string macro TEST_CLI_CRT_EC_PEM tests/data_files/cli2.crt */ #define TEST_CLI_CRT_EC_PEM \ "-----BEGIN CERTIFICATE-----\r\n" \ - "MIICLDCCAbKgAwIBAgIBDTAKBggqhkjOPQQDAjA+MQswCQYDVQQGEwJOTDERMA8G\r\n" \ - "A1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0EwHhcN\r\n" \ - "MTMwOTI0MTU1MjA0WhcNMjMwOTIyMTU1MjA0WjBBMQswCQYDVQQGEwJOTDERMA8G\r\n" \ - "A1UEChMIUG9sYXJTU0wxHzAdBgNVBAMTFlBvbGFyU1NMIFRlc3QgQ2xpZW50IDIw\r\n" \ - "WTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARX5a6xc9/TrLuTuIH/Eq7u5lOszlVT\r\n" \ - "9jQOzC7jYyUL35ji81xgNpbA1RgUcOV/n9VLRRjlsGzVXPiWj4dwo+THo4GdMIGa\r\n" \ - "MAkGA1UdEwQCMAAwHQYDVR0OBBYEFHoAX4Zk/OBd5REQO7LmO8QmP8/iMG4GA1Ud\r\n" \ - "IwRnMGWAFJ1tICRJAT8ry3i1Gbx+JMnb+zZ8oUKkQDA+MQswCQYDVQQGEwJOTDER\r\n" \ - "MA8GA1UEChMIUG9sYXJTU0wxHDAaBgNVBAMTE1BvbGFyc3NsIFRlc3QgRUMgQ0GC\r\n" \ - "CQDBQ+J+YkPM6DAKBggqhkjOPQQDAgNoADBlAjBKZQ17IIOimbmoD/yN7o89u3BM\r\n" \ - "lgOsjnhw3fIOoLIWy2WOGsk/LGF++DzvrRzuNiACMQCd8iem1XS4JK7haj8xocpU\r\n" \ - "LwjQje5PDGHfd3h9tP38Qknu5bJqws0md2KOKHyeV0U=\r\n" \ + "MIIB3zCCAWOgAwIBAgIBDTAMBggqhkjOPQQDAgUAMD4xCzAJBgNVBAYTAk5MMREw\r\n" \ + "DwYDVQQKDAhQb2xhclNTTDEcMBoGA1UEAwwTUG9sYXJTU0wgVGVzdCBFQyBDQTAe\r\n" \ + "Fw0xOTAyMTAxNDQ0MDBaFw0yOTAyMTAxNDQ0MDBaMEExCzAJBgNVBAYTAk5MMREw\r\n" \ + "DwYDVQQKDAhQb2xhclNTTDEfMB0GA1UEAwwWUG9sYXJTU0wgVGVzdCBDbGllbnQg\r\n" \ + "MjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFflrrFz39Osu5O4gf8Sru7mU6zO\r\n" \ + "VVP2NA7MLuNjJQvfmOLzXGA2lsDVGBRw5X+f1UtFGOWwbNVc+JaPh3Cj5MejTTBL\r\n" \ + "MAkGA1UdEwQCMAAwHQYDVR0OBBYEFHoAX4Zk/OBd5REQO7LmO8QmP8/iMB8GA1Ud\r\n" \ + "IwQYMBaAFJ1tICRJAT8ry3i1Gbx+JMnb+zZ8MAwGCCqGSM49BAMCBQADaAAwZQIx\r\n" \ + "AMqme4DKMldUlplDET9Q6Eptre7uUWKhsLOF+zPkKDlfzpIkJYEFgcloDHGYw80u\r\n" \ + "IgIwNftyPXsabTqMM7iEHgVpX/GRozKklY9yQI/5eoA6gGW7Y+imuGR/oao5ySOb\r\n" \ + "a9Vk\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ /* This is generated from tests/data_files/cli2.crt.der using `xxd -i`. */ /* BEGIN FILE binary macro TEST_CLI_CRT_EC_DER tests/data_files/cli2.crt.der */ #define TEST_CLI_CRT_EC_DER { \ - 0x30, 0x82, 0x02, 0x2c, 0x30, 0x82, 0x01, 0xb2, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x0d, 0x30, 0x0a, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, \ - 0x3d, 0x04, 0x03, 0x02, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ - 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, 0x20, 0x54, 0x65, \ - 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x33, 0x30, 0x39, 0x32, 0x34, 0x31, 0x35, 0x35, 0x32, 0x30, 0x34, \ - 0x5a, 0x17, 0x0d, 0x32, 0x33, 0x30, 0x39, 0x32, 0x32, 0x31, 0x35, 0x35, \ - 0x32, 0x30, 0x34, 0x5a, 0x30, 0x41, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x1f, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x04, 0x03, 0x13, \ - 0x16, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x54, 0x65, \ - 0x73, 0x74, 0x20, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x32, 0x30, \ - 0x59, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x02, 0x01, \ - 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07, 0x03, 0x42, \ - 0x00, 0x04, 0x57, 0xe5, 0xae, 0xb1, 0x73, 0xdf, 0xd3, 0xac, 0xbb, 0x93, \ - 0xb8, 0x81, 0xff, 0x12, 0xae, 0xee, 0xe6, 0x53, 0xac, 0xce, 0x55, 0x53, \ - 0xf6, 0x34, 0x0e, 0xcc, 0x2e, 0xe3, 0x63, 0x25, 0x0b, 0xdf, 0x98, 0xe2, \ - 0xf3, 0x5c, 0x60, 0x36, 0x96, 0xc0, 0xd5, 0x18, 0x14, 0x70, 0xe5, 0x7f, \ - 0x9f, 0xd5, 0x4b, 0x45, 0x18, 0xe5, 0xb0, 0x6c, 0xd5, 0x5c, 0xf8, 0x96, \ - 0x8f, 0x87, 0x70, 0xa3, 0xe4, 0xc7, 0xa3, 0x81, 0x9d, 0x30, 0x81, 0x9a, \ - 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, \ - 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x7a, 0x00, \ - 0x5f, 0x86, 0x64, 0xfc, 0xe0, 0x5d, 0xe5, 0x11, 0x10, 0x3b, 0xb2, 0xe6, \ - 0x3b, 0xc4, 0x26, 0x3f, 0xcf, 0xe2, 0x30, 0x6e, 0x06, 0x03, 0x55, 0x1d, \ - 0x23, 0x04, 0x67, 0x30, 0x65, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, \ - 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, \ - 0xfb, 0x36, 0x7c, 0xa1, 0x42, 0xa4, 0x40, 0x30, 0x3e, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x13, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x13, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x73, 0x73, 0x6c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x82, \ - 0x09, 0x00, 0xc1, 0x43, 0xe2, 0x7e, 0x62, 0x43, 0xcc, 0xe8, 0x30, 0x0a, \ - 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x04, 0x03, 0x02, 0x03, 0x68, \ - 0x00, 0x30, 0x65, 0x02, 0x30, 0x4a, 0x65, 0x0d, 0x7b, 0x20, 0x83, 0xa2, \ - 0x99, 0xb9, 0xa8, 0x0f, 0xfc, 0x8d, 0xee, 0x8f, 0x3d, 0xbb, 0x70, 0x4c, \ - 0x96, 0x03, 0xac, 0x8e, 0x78, 0x70, 0xdd, 0xf2, 0x0e, 0xa0, 0xb2, 0x16, \ - 0xcb, 0x65, 0x8e, 0x1a, 0xc9, 0x3f, 0x2c, 0x61, 0x7e, 0xf8, 0x3c, 0xef, \ - 0xad, 0x1c, 0xee, 0x36, 0x20, 0x02, 0x31, 0x00, 0x9d, 0xf2, 0x27, 0xa6, \ - 0xd5, 0x74, 0xb8, 0x24, 0xae, 0xe1, 0x6a, 0x3f, 0x31, 0xa1, 0xca, 0x54, \ - 0x2f, 0x08, 0xd0, 0x8d, 0xee, 0x4f, 0x0c, 0x61, 0xdf, 0x77, 0x78, 0x7d, \ - 0xb4, 0xfd, 0xfc, 0x42, 0x49, 0xee, 0xe5, 0xb2, 0x6a, 0xc2, 0xcd, 0x26, \ - 0x77, 0x62, 0x8e, 0x28, 0x7c, 0x9e, 0x57, 0x45 \ + 0x30, 0x82, 0x01, 0xdf, 0x30, 0x82, 0x01, 0x63, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x0d, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, \ + 0x3d, 0x04, 0x03, 0x02, 0x05, 0x00, 0x30, 0x3e, 0x31, 0x0b, 0x30, 0x09, \ + 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, \ + 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, \ + 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1c, 0x30, 0x1a, 0x06, 0x03, 0x55, 0x04, \ + 0x03, 0x0c, 0x13, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, \ + 0x54, 0x65, 0x73, 0x74, 0x20, 0x45, 0x43, 0x20, 0x43, 0x41, 0x30, 0x1e, \ + 0x17, 0x0d, 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, \ + 0x30, 0x30, 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, \ + 0x34, 0x34, 0x34, 0x30, 0x30, 0x5a, 0x30, 0x41, 0x31, 0x0b, 0x30, 0x09, \ + 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, \ + 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, \ + 0x72, 0x53, 0x53, 0x4c, 0x31, 0x1f, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x04, \ + 0x03, 0x0c, 0x16, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, \ + 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, \ + 0x32, 0x30, 0x59, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, 0xce, 0x3d, \ + 0x02, 0x01, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, 0x03, 0x01, 0x07, \ + 0x03, 0x42, 0x00, 0x04, 0x57, 0xe5, 0xae, 0xb1, 0x73, 0xdf, 0xd3, 0xac, \ + 0xbb, 0x93, 0xb8, 0x81, 0xff, 0x12, 0xae, 0xee, 0xe6, 0x53, 0xac, 0xce, \ + 0x55, 0x53, 0xf6, 0x34, 0x0e, 0xcc, 0x2e, 0xe3, 0x63, 0x25, 0x0b, 0xdf, \ + 0x98, 0xe2, 0xf3, 0x5c, 0x60, 0x36, 0x96, 0xc0, 0xd5, 0x18, 0x14, 0x70, \ + 0xe5, 0x7f, 0x9f, 0xd5, 0x4b, 0x45, 0x18, 0xe5, 0xb0, 0x6c, 0xd5, 0x5c, \ + 0xf8, 0x96, 0x8f, 0x87, 0x70, 0xa3, 0xe4, 0xc7, 0xa3, 0x4d, 0x30, 0x4b, \ + 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, 0x02, 0x30, 0x00, 0x30, \ + 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, 0x04, 0x14, 0x7a, 0x00, \ + 0x5f, 0x86, 0x64, 0xfc, 0xe0, 0x5d, 0xe5, 0x11, 0x10, 0x3b, 0xb2, 0xe6, \ + 0x3b, 0xc4, 0x26, 0x3f, 0xcf, 0xe2, 0x30, 0x1f, 0x06, 0x03, 0x55, 0x1d, \ + 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0x9d, 0x6d, 0x20, 0x24, 0x49, \ + 0x01, 0x3f, 0x2b, 0xcb, 0x78, 0xb5, 0x19, 0xbc, 0x7e, 0x24, 0xc9, 0xdb, \ + 0xfb, 0x36, 0x7c, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0xce, 0x3d, \ + 0x04, 0x03, 0x02, 0x05, 0x00, 0x03, 0x68, 0x00, 0x30, 0x65, 0x02, 0x31, \ + 0x00, 0xca, 0xa6, 0x7b, 0x80, 0xca, 0x32, 0x57, 0x54, 0x96, 0x99, 0x43, \ + 0x11, 0x3f, 0x50, 0xe8, 0x4a, 0x6d, 0xad, 0xee, 0xee, 0x51, 0x62, 0xa1, \ + 0xb0, 0xb3, 0x85, 0xfb, 0x33, 0xe4, 0x28, 0x39, 0x5f, 0xce, 0x92, 0x24, \ + 0x25, 0x81, 0x05, 0x81, 0xc9, 0x68, 0x0c, 0x71, 0x98, 0xc3, 0xcd, 0x2e, \ + 0x22, 0x02, 0x30, 0x35, 0xfb, 0x72, 0x3d, 0x7b, 0x1a, 0x6d, 0x3a, 0x8c, \ + 0x33, 0xb8, 0x84, 0x1e, 0x05, 0x69, 0x5f, 0xf1, 0x91, 0xa3, 0x32, 0xa4, \ + 0x95, 0x8f, 0x72, 0x40, 0x8f, 0xf9, 0x7a, 0x80, 0x3a, 0x80, 0x65, 0xbb, \ + 0x63, 0xe8, 0xa6, 0xb8, 0x64, 0x7f, 0xa1, 0xaa, 0x39, 0xc9, 0x23, 0x9b, \ + 0x6b, 0xd5, 0x64 \ } /* END FILE */ @@ -1067,7 +1052,7 @@ "-----BEGIN CERTIFICATE-----\r\n" \ "MIIDPzCCAiegAwIBAgIBBDANBgkqhkiG9w0BAQsFADA7MQswCQYDVQQGEwJOTDER\r\n" \ "MA8GA1UECgwIUG9sYXJTU0wxGTAXBgNVBAMMEFBvbGFyU1NMIFRlc3QgQ0EwHhcN\r\n" \ - "MTEwMjEyMTQ0NDA2WhcNMjEwMjEyMTQ0NDA2WjA8MQswCQYDVQQGEwJOTDERMA8G\r\n" \ + "MTkwMjEwMTQ0NDA2WhcNMjkwMjEwMTQ0NDA2WjA8MQswCQYDVQQGEwJOTDERMA8G\r\n" \ "A1UECgwIUG9sYXJTU0wxGjAYBgNVBAMMEVBvbGFyU1NMIENsaWVudCAyMIIBIjAN\r\n" \ "BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyHTEzLn5tXnpRdkUYLB9u5Pyax6f\r\n" \ "M60Nj4o8VmXl3ETZzGaFB9X4J7BKNdBjngpuG7fa8H6r7gwQk4ZJGDTzqCrSV/Uu\r\n" \ @@ -1077,12 +1062,12 @@ "/DZrtenNLQNiTrM9AM+vdqBpVoNq0qjU51Bx5rU2BXcFbXvI5MT9TNUhXwIDAQAB\r\n" \ "o00wSzAJBgNVHRMEAjAAMB0GA1UdDgQWBBRxoQBzckAvVHZeM/xSj7zx3WtGITAf\r\n" \ "BgNVHSMEGDAWgBS0WuSls97SUva51aaVD+s+vMf9/zANBgkqhkiG9w0BAQsFAAOC\r\n" \ - "AQEAlHabem2Tu69VUN7EipwnQn1dIHdgvT5i+iQHpSxY1crPnBbAeSdAXwsVEqLQ\r\n" \ - "gOOIAQD5VIITNuoGgo4i+4OpNh9u7ZkpRHla+/swsfrFWRRbBNP5Bcu74AGLstwU\r\n" \ - "zM8gIkBiyfM1Q1qDQISV9trlCG6O8vh8dp/rbI3rfzo99BOHXgFCrzXjCuW4vDsF\r\n" \ - "r+Dao26bX3sJ6UnEWg1H3o2x6PpUcvQ36h71/bz4TEbbUUEpe02V4QWuL+wrhHJL\r\n" \ - "U7o3SVE3Og7jPF8sat0a50YUWhwEFI256m02KAXLg89ueUyYKEr6rNwhcvXJpvU9\r\n" \ - "giIVvd0Sbjjnn7NC4VDbcXV8vw==\r\n" \ + "AQEAXidv1d4pLlBiKWED95rMycBdgDcgyNqJxakFkRfRyA2y1mlyTn7uBXRkNLY5\r\n" \ + "ZFzK82GCjk2Q2OD4RZSCPAJJqLpHHU34t71ciffvy2KK81YvrxczRhMAE64i+qna\r\n" \ + "yP3Td2XuWJR05PVPoSemsNELs9gWttdnYy3ce+EY2Y0n7Rsi7982EeLIAA7H6ca4\r\n" \ + "2Es/NUH//JZJT32OP0doMxeDRA+vplkKqTLLWf7dX26LIriBkBaRCgR5Yv9LBPFc\r\n" \ + "NOtpzu/LbrY7QFXKJMI+JXDudCsOn8KCmiA4d6Emisqfh3V3485l7HEQNcvLTxlD\r\n" \ + "6zDQyi0/ykYUYZkwQTK1N2Nvlw==\r\n" \ "-----END CERTIFICATE-----\r\n" /* END FILE */ @@ -1090,76 +1075,76 @@ using `xxd -i.` */ /* BEGIN FILE binary macro TEST_CLI_CRT_RSA_DER tests/data_files/cli-rsa-sha256.crt.der */ #define TEST_CLI_CRT_RSA_DER { \ - 0x30, 0x82, 0x03, 0x3f, 0x30, 0x82, 0x02, 0x27, 0xa0, 0x03, 0x02, 0x01, \ - 0x02, 0x02, 0x01, 0x04, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ - 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ - 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ - 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ - 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ - 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ - 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ - 0x31, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ - 0x5a, 0x17, 0x0d, 0x32, 0x31, 0x30, 0x32, 0x31, 0x32, 0x31, 0x34, 0x34, \ - 0x34, 0x30, 0x36, 0x5a, 0x30, 0x3c, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ - 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ - 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ - 0x53, 0x4c, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ - 0x11, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x43, 0x6c, \ - 0x69, 0x65, 0x6e, 0x74, 0x20, 0x32, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, \ - 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, \ - 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, \ - 0x01, 0x01, 0x00, 0xc8, 0x74, 0xc4, 0xcc, 0xb9, 0xf9, 0xb5, 0x79, 0xe9, \ - 0x45, 0xd9, 0x14, 0x60, 0xb0, 0x7d, 0xbb, 0x93, 0xf2, 0x6b, 0x1e, 0x9f, \ - 0x33, 0xad, 0x0d, 0x8f, 0x8a, 0x3c, 0x56, 0x65, 0xe5, 0xdc, 0x44, 0xd9, \ - 0xcc, 0x66, 0x85, 0x07, 0xd5, 0xf8, 0x27, 0xb0, 0x4a, 0x35, 0xd0, 0x63, \ - 0x9e, 0x0a, 0x6e, 0x1b, 0xb7, 0xda, 0xf0, 0x7e, 0xab, 0xee, 0x0c, 0x10, \ - 0x93, 0x86, 0x49, 0x18, 0x34, 0xf3, 0xa8, 0x2a, 0xd2, 0x57, 0xf5, 0x2e, \ - 0xd4, 0x2f, 0x77, 0x29, 0x84, 0x61, 0x4d, 0x82, 0x50, 0x8f, 0xa7, 0x95, \ - 0x48, 0x70, 0xf5, 0x6e, 0x4d, 0xb2, 0xd5, 0x13, 0xc3, 0xd2, 0x1a, 0xed, \ - 0xe6, 0x43, 0xea, 0x42, 0x14, 0xeb, 0x74, 0xea, 0xc0, 0xed, 0x1f, 0xd4, \ - 0x57, 0x4e, 0xa9, 0xf3, 0xa8, 0xed, 0xd2, 0xe0, 0xc1, 0x30, 0x71, 0x30, \ - 0x32, 0x30, 0xd5, 0xd3, 0xf6, 0x08, 0xd0, 0x56, 0x4f, 0x46, 0x8e, 0xf2, \ - 0x5f, 0xf9, 0x3d, 0x67, 0x91, 0x88, 0x30, 0x2e, 0x42, 0xb2, 0xdf, 0x7d, \ - 0xfb, 0xe5, 0x0c, 0x77, 0xff, 0xec, 0x31, 0xc0, 0x78, 0x8f, 0xbf, 0xc2, \ - 0x7f, 0xca, 0xad, 0x6c, 0x21, 0xd6, 0x8d, 0xd9, 0x8b, 0x6a, 0x8e, 0x6f, \ - 0xe0, 0x9b, 0xf8, 0x10, 0x56, 0xcc, 0xb3, 0x8e, 0x13, 0x15, 0xe6, 0x34, \ - 0x04, 0x66, 0xc7, 0xee, 0xf9, 0x36, 0x0e, 0x6a, 0x95, 0xf6, 0x09, 0x9a, \ - 0x06, 0x67, 0xf4, 0x65, 0x71, 0xf8, 0xca, 0xa4, 0xb1, 0x25, 0xe0, 0xfe, \ - 0x3c, 0x8b, 0x35, 0x04, 0x67, 0xba, 0xe0, 0x4f, 0x76, 0x85, 0xfc, 0x7f, \ - 0xfc, 0x36, 0x6b, 0xb5, 0xe9, 0xcd, 0x2d, 0x03, 0x62, 0x4e, 0xb3, 0x3d, \ - 0x00, 0xcf, 0xaf, 0x76, 0xa0, 0x69, 0x56, 0x83, 0x6a, 0xd2, 0xa8, 0xd4, \ - 0xe7, 0x50, 0x71, 0xe6, 0xb5, 0x36, 0x05, 0x77, 0x05, 0x6d, 0x7b, 0xc8, \ - 0xe4, 0xc4, 0xfd, 0x4c, 0xd5, 0x21, 0x5f, 0x02, 0x03, 0x01, 0x00, 0x01, \ - 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, \ - 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, \ - 0x04, 0x14, 0x71, 0xa1, 0x00, 0x73, 0x72, 0x40, 0x2f, 0x54, 0x76, 0x5e, \ - 0x33, 0xfc, 0x52, 0x8f, 0xbc, 0xf1, 0xdd, 0x6b, 0x46, 0x21, 0x30, 0x1f, \ - 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0xb4, \ - 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, 0xa6, 0x95, \ - 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, 0x09, 0x2a, \ - 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x03, 0x82, \ - 0x01, 0x01, 0x00, 0x94, 0x76, 0x9b, 0x7a, 0x6d, 0x93, 0xbb, 0xaf, 0x55, \ - 0x50, 0xde, 0xc4, 0x8a, 0x9c, 0x27, 0x42, 0x7d, 0x5d, 0x20, 0x77, 0x60, \ - 0xbd, 0x3e, 0x62, 0xfa, 0x24, 0x07, 0xa5, 0x2c, 0x58, 0xd5, 0xca, 0xcf, \ - 0x9c, 0x16, 0xc0, 0x79, 0x27, 0x40, 0x5f, 0x0b, 0x15, 0x12, 0xa2, 0xd0, \ - 0x80, 0xe3, 0x88, 0x01, 0x00, 0xf9, 0x54, 0x82, 0x13, 0x36, 0xea, 0x06, \ - 0x82, 0x8e, 0x22, 0xfb, 0x83, 0xa9, 0x36, 0x1f, 0x6e, 0xed, 0x99, 0x29, \ - 0x44, 0x79, 0x5a, 0xfb, 0xfb, 0x30, 0xb1, 0xfa, 0xc5, 0x59, 0x14, 0x5b, \ - 0x04, 0xd3, 0xf9, 0x05, 0xcb, 0xbb, 0xe0, 0x01, 0x8b, 0xb2, 0xdc, 0x14, \ - 0xcc, 0xcf, 0x20, 0x22, 0x40, 0x62, 0xc9, 0xf3, 0x35, 0x43, 0x5a, 0x83, \ - 0x40, 0x84, 0x95, 0xf6, 0xda, 0xe5, 0x08, 0x6e, 0x8e, 0xf2, 0xf8, 0x7c, \ - 0x76, 0x9f, 0xeb, 0x6c, 0x8d, 0xeb, 0x7f, 0x3a, 0x3d, 0xf4, 0x13, 0x87, \ - 0x5e, 0x01, 0x42, 0xaf, 0x35, 0xe3, 0x0a, 0xe5, 0xb8, 0xbc, 0x3b, 0x05, \ - 0xaf, 0xe0, 0xda, 0xa3, 0x6e, 0x9b, 0x5f, 0x7b, 0x09, 0xe9, 0x49, 0xc4, \ - 0x5a, 0x0d, 0x47, 0xde, 0x8d, 0xb1, 0xe8, 0xfa, 0x54, 0x72, 0xf4, 0x37, \ - 0xea, 0x1e, 0xf5, 0xfd, 0xbc, 0xf8, 0x4c, 0x46, 0xdb, 0x51, 0x41, 0x29, \ - 0x7b, 0x4d, 0x95, 0xe1, 0x05, 0xae, 0x2f, 0xec, 0x2b, 0x84, 0x72, 0x4b, \ - 0x53, 0xba, 0x37, 0x49, 0x51, 0x37, 0x3a, 0x0e, 0xe3, 0x3c, 0x5f, 0x2c, \ - 0x6a, 0xdd, 0x1a, 0xe7, 0x46, 0x14, 0x5a, 0x1c, 0x04, 0x14, 0x8d, 0xb9, \ - 0xea, 0x6d, 0x36, 0x28, 0x05, 0xcb, 0x83, 0xcf, 0x6e, 0x79, 0x4c, 0x98, \ - 0x28, 0x4a, 0xfa, 0xac, 0xdc, 0x21, 0x72, 0xf5, 0xc9, 0xa6, 0xf5, 0x3d, \ - 0x82, 0x22, 0x15, 0xbd, 0xdd, 0x12, 0x6e, 0x38, 0xe7, 0x9f, 0xb3, 0x42, \ - 0xe1, 0x50, 0xdb, 0x71, 0x75, 0x7c, 0xbf \ + 0x30, 0x82, 0x03, 0x3f, 0x30, 0x82, 0x02, 0x27, 0xa0, 0x03, 0x02, 0x01, \ + 0x02, 0x02, 0x01, 0x04, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, \ + 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x30, 0x3b, 0x31, 0x0b, 0x30, \ + 0x09, 0x06, 0x03, 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, \ + 0x30, 0x0f, 0x06, 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, \ + 0x61, 0x72, 0x53, 0x53, 0x4c, 0x31, 0x19, 0x30, 0x17, 0x06, 0x03, 0x55, \ + 0x04, 0x03, 0x0c, 0x10, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, \ + 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x43, 0x41, 0x30, 0x1e, 0x17, 0x0d, \ + 0x31, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, 0x34, 0x30, 0x36, \ + 0x5a, 0x17, 0x0d, 0x32, 0x39, 0x30, 0x32, 0x31, 0x30, 0x31, 0x34, 0x34, \ + 0x34, 0x30, 0x36, 0x5a, 0x30, 0x3c, 0x31, 0x0b, 0x30, 0x09, 0x06, 0x03, \ + 0x55, 0x04, 0x06, 0x13, 0x02, 0x4e, 0x4c, 0x31, 0x11, 0x30, 0x0f, 0x06, \ + 0x03, 0x55, 0x04, 0x0a, 0x0c, 0x08, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, \ + 0x53, 0x4c, 0x31, 0x1a, 0x30, 0x18, 0x06, 0x03, 0x55, 0x04, 0x03, 0x0c, \ + 0x11, 0x50, 0x6f, 0x6c, 0x61, 0x72, 0x53, 0x53, 0x4c, 0x20, 0x43, 0x6c, \ + 0x69, 0x65, 0x6e, 0x74, 0x20, 0x32, 0x30, 0x82, 0x01, 0x22, 0x30, 0x0d, \ + 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, \ + 0x00, 0x03, 0x82, 0x01, 0x0f, 0x00, 0x30, 0x82, 0x01, 0x0a, 0x02, 0x82, \ + 0x01, 0x01, 0x00, 0xc8, 0x74, 0xc4, 0xcc, 0xb9, 0xf9, 0xb5, 0x79, 0xe9, \ + 0x45, 0xd9, 0x14, 0x60, 0xb0, 0x7d, 0xbb, 0x93, 0xf2, 0x6b, 0x1e, 0x9f, \ + 0x33, 0xad, 0x0d, 0x8f, 0x8a, 0x3c, 0x56, 0x65, 0xe5, 0xdc, 0x44, 0xd9, \ + 0xcc, 0x66, 0x85, 0x07, 0xd5, 0xf8, 0x27, 0xb0, 0x4a, 0x35, 0xd0, 0x63, \ + 0x9e, 0x0a, 0x6e, 0x1b, 0xb7, 0xda, 0xf0, 0x7e, 0xab, 0xee, 0x0c, 0x10, \ + 0x93, 0x86, 0x49, 0x18, 0x34, 0xf3, 0xa8, 0x2a, 0xd2, 0x57, 0xf5, 0x2e, \ + 0xd4, 0x2f, 0x77, 0x29, 0x84, 0x61, 0x4d, 0x82, 0x50, 0x8f, 0xa7, 0x95, \ + 0x48, 0x70, 0xf5, 0x6e, 0x4d, 0xb2, 0xd5, 0x13, 0xc3, 0xd2, 0x1a, 0xed, \ + 0xe6, 0x43, 0xea, 0x42, 0x14, 0xeb, 0x74, 0xea, 0xc0, 0xed, 0x1f, 0xd4, \ + 0x57, 0x4e, 0xa9, 0xf3, 0xa8, 0xed, 0xd2, 0xe0, 0xc1, 0x30, 0x71, 0x30, \ + 0x32, 0x30, 0xd5, 0xd3, 0xf6, 0x08, 0xd0, 0x56, 0x4f, 0x46, 0x8e, 0xf2, \ + 0x5f, 0xf9, 0x3d, 0x67, 0x91, 0x88, 0x30, 0x2e, 0x42, 0xb2, 0xdf, 0x7d, \ + 0xfb, 0xe5, 0x0c, 0x77, 0xff, 0xec, 0x31, 0xc0, 0x78, 0x8f, 0xbf, 0xc2, \ + 0x7f, 0xca, 0xad, 0x6c, 0x21, 0xd6, 0x8d, 0xd9, 0x8b, 0x6a, 0x8e, 0x6f, \ + 0xe0, 0x9b, 0xf8, 0x10, 0x56, 0xcc, 0xb3, 0x8e, 0x13, 0x15, 0xe6, 0x34, \ + 0x04, 0x66, 0xc7, 0xee, 0xf9, 0x36, 0x0e, 0x6a, 0x95, 0xf6, 0x09, 0x9a, \ + 0x06, 0x67, 0xf4, 0x65, 0x71, 0xf8, 0xca, 0xa4, 0xb1, 0x25, 0xe0, 0xfe, \ + 0x3c, 0x8b, 0x35, 0x04, 0x67, 0xba, 0xe0, 0x4f, 0x76, 0x85, 0xfc, 0x7f, \ + 0xfc, 0x36, 0x6b, 0xb5, 0xe9, 0xcd, 0x2d, 0x03, 0x62, 0x4e, 0xb3, 0x3d, \ + 0x00, 0xcf, 0xaf, 0x76, 0xa0, 0x69, 0x56, 0x83, 0x6a, 0xd2, 0xa8, 0xd4, \ + 0xe7, 0x50, 0x71, 0xe6, 0xb5, 0x36, 0x05, 0x77, 0x05, 0x6d, 0x7b, 0xc8, \ + 0xe4, 0xc4, 0xfd, 0x4c, 0xd5, 0x21, 0x5f, 0x02, 0x03, 0x01, 0x00, 0x01, \ + 0xa3, 0x4d, 0x30, 0x4b, 0x30, 0x09, 0x06, 0x03, 0x55, 0x1d, 0x13, 0x04, \ + 0x02, 0x30, 0x00, 0x30, 0x1d, 0x06, 0x03, 0x55, 0x1d, 0x0e, 0x04, 0x16, \ + 0x04, 0x14, 0x71, 0xa1, 0x00, 0x73, 0x72, 0x40, 0x2f, 0x54, 0x76, 0x5e, \ + 0x33, 0xfc, 0x52, 0x8f, 0xbc, 0xf1, 0xdd, 0x6b, 0x46, 0x21, 0x30, 0x1f, \ + 0x06, 0x03, 0x55, 0x1d, 0x23, 0x04, 0x18, 0x30, 0x16, 0x80, 0x14, 0xb4, \ + 0x5a, 0xe4, 0xa5, 0xb3, 0xde, 0xd2, 0x52, 0xf6, 0xb9, 0xd5, 0xa6, 0x95, \ + 0x0f, 0xeb, 0x3e, 0xbc, 0xc7, 0xfd, 0xff, 0x30, 0x0d, 0x06, 0x09, 0x2a, \ + 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b, 0x05, 0x00, 0x03, 0x82, \ + 0x01, 0x01, 0x00, 0x5e, 0x27, 0x6f, 0xd5, 0xde, 0x29, 0x2e, 0x50, 0x62, \ + 0x29, 0x61, 0x03, 0xf7, 0x9a, 0xcc, 0xc9, 0xc0, 0x5d, 0x80, 0x37, 0x20, \ + 0xc8, 0xda, 0x89, 0xc5, 0xa9, 0x05, 0x91, 0x17, 0xd1, 0xc8, 0x0d, 0xb2, \ + 0xd6, 0x69, 0x72, 0x4e, 0x7e, 0xee, 0x05, 0x74, 0x64, 0x34, 0xb6, 0x39, \ + 0x64, 0x5c, 0xca, 0xf3, 0x61, 0x82, 0x8e, 0x4d, 0x90, 0xd8, 0xe0, 0xf8, \ + 0x45, 0x94, 0x82, 0x3c, 0x02, 0x49, 0xa8, 0xba, 0x47, 0x1d, 0x4d, 0xf8, \ + 0xb7, 0xbd, 0x5c, 0x89, 0xf7, 0xef, 0xcb, 0x62, 0x8a, 0xf3, 0x56, 0x2f, \ + 0xaf, 0x17, 0x33, 0x46, 0x13, 0x00, 0x13, 0xae, 0x22, 0xfa, 0xa9, 0xda, \ + 0xc8, 0xfd, 0xd3, 0x77, 0x65, 0xee, 0x58, 0x94, 0x74, 0xe4, 0xf5, 0x4f, \ + 0xa1, 0x27, 0xa6, 0xb0, 0xd1, 0x0b, 0xb3, 0xd8, 0x16, 0xb6, 0xd7, 0x67, \ + 0x63, 0x2d, 0xdc, 0x7b, 0xe1, 0x18, 0xd9, 0x8d, 0x27, 0xed, 0x1b, 0x22, \ + 0xef, 0xdf, 0x36, 0x11, 0xe2, 0xc8, 0x00, 0x0e, 0xc7, 0xe9, 0xc6, 0xb8, \ + 0xd8, 0x4b, 0x3f, 0x35, 0x41, 0xff, 0xfc, 0x96, 0x49, 0x4f, 0x7d, 0x8e, \ + 0x3f, 0x47, 0x68, 0x33, 0x17, 0x83, 0x44, 0x0f, 0xaf, 0xa6, 0x59, 0x0a, \ + 0xa9, 0x32, 0xcb, 0x59, 0xfe, 0xdd, 0x5f, 0x6e, 0x8b, 0x22, 0xb8, 0x81, \ + 0x90, 0x16, 0x91, 0x0a, 0x04, 0x79, 0x62, 0xff, 0x4b, 0x04, 0xf1, 0x5c, \ + 0x34, 0xeb, 0x69, 0xce, 0xef, 0xcb, 0x6e, 0xb6, 0x3b, 0x40, 0x55, 0xca, \ + 0x24, 0xc2, 0x3e, 0x25, 0x70, 0xee, 0x74, 0x2b, 0x0e, 0x9f, 0xc2, 0x82, \ + 0x9a, 0x20, 0x38, 0x77, 0xa1, 0x26, 0x8a, 0xca, 0x9f, 0x87, 0x75, 0x77, \ + 0xe3, 0xce, 0x65, 0xec, 0x71, 0x10, 0x35, 0xcb, 0xcb, 0x4f, 0x19, 0x43, \ + 0xeb, 0x30, 0xd0, 0xca, 0x2d, 0x3f, 0xca, 0x46, 0x14, 0x61, 0x99, 0x30, \ + 0x41, 0x32, 0xb5, 0x37, 0x63, 0x6f, 0x97 \ } /* END FILE */ diff --git a/thirdparty/mbedtls/library/ecdsa.c b/thirdparty/mbedtls/library/ecdsa.c index dc19384d614..2b4800642d7 100644 --- a/thirdparty/mbedtls/library/ecdsa.c +++ b/thirdparty/mbedtls/library/ecdsa.c @@ -172,11 +172,11 @@ static void ecdsa_restart_det_free( mbedtls_ecdsa_restart_det_ctx *ctx ) } #endif /* MBEDTLS_ECDSA_DETERMINISTIC */ -#define ECDSA_RS_ECP &rs_ctx->ecp +#define ECDSA_RS_ECP ( rs_ctx == NULL ? NULL : &rs_ctx->ecp ) /* Utility macro for checking and updating ops budget */ #define ECDSA_BUDGET( ops ) \ - MBEDTLS_MPI_CHK( mbedtls_ecp_check_budget( grp, &rs_ctx->ecp, ops ) ); + MBEDTLS_MPI_CHK( mbedtls_ecp_check_budget( grp, ECDSA_RS_ECP, ops ) ); /* Call this when entering a function that needs its own sub-context */ #define ECDSA_RS_ENTER( SUB ) do { \ @@ -254,6 +254,8 @@ static int ecdsa_sign_restartable( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, const mbedtls_mpi *d, const unsigned char *buf, size_t blen, int (*f_rng)(void *, unsigned char *, size_t), void *p_rng, + int (*f_rng_blind)(void *, unsigned char *, size_t), + void *p_rng_blind, mbedtls_ecdsa_restart_ctx *rs_ctx ) { int ret, key_tries, sign_tries; @@ -323,7 +325,9 @@ static int ecdsa_sign_restartable( mbedtls_ecp_group *grp, mul: #endif MBEDTLS_MPI_CHK( mbedtls_ecp_mul_restartable( grp, &R, pk, &grp->G, - f_rng, p_rng, ECDSA_RS_ECP ) ); + f_rng_blind, + p_rng_blind, + ECDSA_RS_ECP ) ); MBEDTLS_MPI_CHK( mbedtls_mpi_mod_mpi( pr, &R.X, &grp->N ) ); } while( mbedtls_mpi_cmp_int( pr, 0 ) == 0 ); @@ -349,7 +353,8 @@ modn: * Generate a random value to blind inv_mod in next step, * avoiding a potential timing leak. */ - MBEDTLS_MPI_CHK( mbedtls_ecp_gen_privkey( grp, &t, f_rng, p_rng ) ); + MBEDTLS_MPI_CHK( mbedtls_ecp_gen_privkey( grp, &t, f_rng_blind, + p_rng_blind ) ); /* * Step 6: compute s = (e + r * d) / k = t (e + rd) / (kt) mod n @@ -392,8 +397,9 @@ int mbedtls_ecdsa_sign( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, ECDSA_VALIDATE_RET( f_rng != NULL ); ECDSA_VALIDATE_RET( buf != NULL || blen == 0 ); + /* Use the same RNG for both blinding and ephemeral key generation */ return( ecdsa_sign_restartable( grp, r, s, d, buf, blen, - f_rng, p_rng, NULL ) ); + f_rng, p_rng, f_rng, p_rng, NULL ) ); } #endif /* !MBEDTLS_ECDSA_SIGN_ALT */ @@ -405,6 +411,8 @@ static int ecdsa_sign_det_restartable( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, const mbedtls_mpi *d, const unsigned char *buf, size_t blen, mbedtls_md_type_t md_alg, + int (*f_rng_blind)(void *, unsigned char *, size_t), + void *p_rng_blind, mbedtls_ecdsa_restart_ctx *rs_ctx ) { int ret; @@ -451,8 +459,70 @@ sign: ret = mbedtls_ecdsa_sign( grp, r, s, d, buf, blen, mbedtls_hmac_drbg_random, p_rng ); #else - ret = ecdsa_sign_restartable( grp, r, s, d, buf, blen, - mbedtls_hmac_drbg_random, p_rng, rs_ctx ); + if( f_rng_blind != NULL ) + ret = ecdsa_sign_restartable( grp, r, s, d, buf, blen, + mbedtls_hmac_drbg_random, p_rng, + f_rng_blind, p_rng_blind, rs_ctx ); + else + { + mbedtls_hmac_drbg_context *p_rng_blind_det; + +#if !defined(MBEDTLS_ECP_RESTARTABLE) + /* + * To avoid reusing rng_ctx and risking incorrect behavior we seed a + * second HMAC-DRBG with the same seed. We also apply a label to avoid + * reusing the bits of the ephemeral key for blinding and eliminate the + * risk that they leak this way. + */ + const char* blind_label = "BLINDING CONTEXT"; + mbedtls_hmac_drbg_context rng_ctx_blind; + + mbedtls_hmac_drbg_init( &rng_ctx_blind ); + p_rng_blind_det = &rng_ctx_blind; + + mbedtls_hmac_drbg_seed_buf( p_rng_blind_det, md_info, + data, 2 * grp_len ); + ret = mbedtls_hmac_drbg_update_ret( p_rng_blind_det, + (const unsigned char*) blind_label, + strlen( blind_label ) ); + if( ret != 0 ) + { + mbedtls_hmac_drbg_free( &rng_ctx_blind ); + goto cleanup; + } +#else + /* + * In the case of restartable computations we would either need to store + * the second RNG in the restart context too or set it up at every + * restart. The first option would penalize the correct application of + * the function and the second would defeat the purpose of the + * restartable feature. + * + * Therefore in this case we reuse the original RNG. This comes with the + * price that the resulting signature might not be a valid deterministic + * ECDSA signature with a very low probability (same magnitude as + * successfully guessing the private key). However even then it is still + * a valid ECDSA signature. + */ + p_rng_blind_det = p_rng; +#endif /* MBEDTLS_ECP_RESTARTABLE */ + + /* + * Since the output of the RNGs is always the same for the same key and + * message, this limits the efficiency of blinding and leaks information + * through side channels. After mbedtls_ecdsa_sign_det() is removed NULL + * won't be a valid value for f_rng_blind anymore. Therefore it should + * be checked by the caller and this branch and check can be removed. + */ + ret = ecdsa_sign_restartable( grp, r, s, d, buf, blen, + mbedtls_hmac_drbg_random, p_rng, + mbedtls_hmac_drbg_random, p_rng_blind_det, + rs_ctx ); + +#if !defined(MBEDTLS_ECP_RESTARTABLE) + mbedtls_hmac_drbg_free( &rng_ctx_blind ); +#endif + } #endif /* MBEDTLS_ECDSA_SIGN_ALT */ cleanup: @@ -465,11 +535,12 @@ cleanup: } /* - * Deterministic signature wrapper + * Deterministic signature wrappers */ -int mbedtls_ecdsa_sign_det( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, - const mbedtls_mpi *d, const unsigned char *buf, size_t blen, - mbedtls_md_type_t md_alg ) +int mbedtls_ecdsa_sign_det( mbedtls_ecp_group *grp, mbedtls_mpi *r, + mbedtls_mpi *s, const mbedtls_mpi *d, + const unsigned char *buf, size_t blen, + mbedtls_md_type_t md_alg ) { ECDSA_VALIDATE_RET( grp != NULL ); ECDSA_VALIDATE_RET( r != NULL ); @@ -477,7 +548,27 @@ int mbedtls_ecdsa_sign_det( mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi ECDSA_VALIDATE_RET( d != NULL ); ECDSA_VALIDATE_RET( buf != NULL || blen == 0 ); - return( ecdsa_sign_det_restartable( grp, r, s, d, buf, blen, md_alg, NULL ) ); + return( ecdsa_sign_det_restartable( grp, r, s, d, buf, blen, md_alg, + NULL, NULL, NULL ) ); +} + +int mbedtls_ecdsa_sign_det_ext( mbedtls_ecp_group *grp, mbedtls_mpi *r, + mbedtls_mpi *s, const mbedtls_mpi *d, + const unsigned char *buf, size_t blen, + mbedtls_md_type_t md_alg, + int (*f_rng_blind)(void *, unsigned char *, + size_t), + void *p_rng_blind ) +{ + ECDSA_VALIDATE_RET( grp != NULL ); + ECDSA_VALIDATE_RET( r != NULL ); + ECDSA_VALIDATE_RET( s != NULL ); + ECDSA_VALIDATE_RET( d != NULL ); + ECDSA_VALIDATE_RET( buf != NULL || blen == 0 ); + ECDSA_VALIDATE_RET( f_rng_blind != NULL ); + + return( ecdsa_sign_det_restartable( grp, r, s, d, buf, blen, md_alg, + f_rng_blind, p_rng_blind, NULL ) ); } #endif /* MBEDTLS_ECDSA_DETERMINISTIC */ @@ -656,11 +747,9 @@ int mbedtls_ecdsa_write_signature_restartable( mbedtls_ecdsa_context *ctx, mbedtls_mpi_init( &s ); #if defined(MBEDTLS_ECDSA_DETERMINISTIC) - (void) f_rng; - (void) p_rng; - MBEDTLS_MPI_CHK( ecdsa_sign_det_restartable( &ctx->grp, &r, &s, &ctx->d, - hash, hlen, md_alg, rs_ctx ) ); + hash, hlen, md_alg, f_rng, + p_rng, rs_ctx ) ); #else (void) md_alg; @@ -668,8 +757,10 @@ int mbedtls_ecdsa_write_signature_restartable( mbedtls_ecdsa_context *ctx, MBEDTLS_MPI_CHK( mbedtls_ecdsa_sign( &ctx->grp, &r, &s, &ctx->d, hash, hlen, f_rng, p_rng ) ); #else + /* Use the same RNG for both blinding and ephemeral key generation */ MBEDTLS_MPI_CHK( ecdsa_sign_restartable( &ctx->grp, &r, &s, &ctx->d, - hash, hlen, f_rng, p_rng, rs_ctx ) ); + hash, hlen, f_rng, p_rng, f_rng, + p_rng, rs_ctx ) ); #endif /* MBEDTLS_ECDSA_SIGN_ALT */ #endif /* MBEDTLS_ECDSA_DETERMINISTIC */ diff --git a/thirdparty/mbedtls/library/ecjpake.c b/thirdparty/mbedtls/library/ecjpake.c index be941b14b11..1845c936abc 100644 --- a/thirdparty/mbedtls/library/ecjpake.c +++ b/thirdparty/mbedtls/library/ecjpake.c @@ -226,7 +226,7 @@ static int ecjpake_hash( const mbedtls_md_info_t *md_info, p += id_len; /* Compute hash */ - mbedtls_md( md_info, buf, p - buf, hash ); + MBEDTLS_MPI_CHK( mbedtls_md( md_info, buf, p - buf, hash ) ); /* Turn it into an integer mod n */ MBEDTLS_MPI_CHK( mbedtls_mpi_read_binary( h, hash, @@ -951,7 +951,7 @@ static const unsigned char ecjpake_test_pms[] = { 0xb4, 0x38, 0xf7, 0x19, 0xd3, 0xc4, 0xf3, 0x51 }; -/* Load my private keys and generate the correponding public keys */ +/* Load my private keys and generate the corresponding public keys */ static int ecjpake_test_load( mbedtls_ecjpake_context *ctx, const unsigned char *xm1, size_t len1, const unsigned char *xm2, size_t len2 ) diff --git a/thirdparty/mbedtls/library/entropy_poll.c b/thirdparty/mbedtls/library/entropy_poll.c index 4556f88a557..ba56b70f77e 100644 --- a/thirdparty/mbedtls/library/entropy_poll.c +++ b/thirdparty/mbedtls/library/entropy_poll.c @@ -61,28 +61,43 @@ #define _WIN32_WINNT 0x0400 #endif #include -#include +#include +#if defined(_MSC_VER) && _MSC_VER <= 1600 +/* Visual Studio 2010 and earlier issue a warning when both and + * are included, as they redefine a number of _MAX constants. + * These constants are guaranteed to be the same, though, so we suppress the + * warning when including intsafe.h. + */ +#pragma warning( push ) +#pragma warning( disable : 4005 ) +#endif +#include +#if defined(_MSC_VER) && _MSC_VER <= 1600 +#pragma warning( pop ) +#endif int mbedtls_platform_entropy_poll( void *data, unsigned char *output, size_t len, size_t *olen ) { - HCRYPTPROV provider; + ULONG len_as_ulong = 0; ((void) data); *olen = 0; - if( CryptAcquireContext( &provider, NULL, NULL, - PROV_RSA_FULL, CRYPT_VERIFYCONTEXT ) == FALSE ) + /* + * BCryptGenRandom takes ULONG for size, which is smaller than size_t on + * 64-bit Windows platforms. Ensure len's value can be safely converted into + * a ULONG. + */ + if ( FAILED( SizeTToULong( len, &len_as_ulong ) ) ) { return( MBEDTLS_ERR_ENTROPY_SOURCE_FAILED ); } - if( CryptGenRandom( provider, (DWORD) len, output ) == FALSE ) + if ( !BCRYPT_SUCCESS( BCryptGenRandom( NULL, output, len_as_ulong, BCRYPT_USE_SYSTEM_PREFERRED_RNG ) ) ) { - CryptReleaseContext( provider, 0 ); return( MBEDTLS_ERR_ENTROPY_SOURCE_FAILED ); } - CryptReleaseContext( provider, 0 ); *olen = len; return( 0 ); diff --git a/thirdparty/mbedtls/library/error.c b/thirdparty/mbedtls/library/error.c index 12312a05623..c596f0bcc57 100644 --- a/thirdparty/mbedtls/library/error.c +++ b/thirdparty/mbedtls/library/error.c @@ -567,7 +567,7 @@ void mbedtls_strerror( int ret, char *buf, size_t buflen ) if( use_ret == -(MBEDTLS_ERR_X509_BUFFER_TOO_SMALL) ) mbedtls_snprintf( buf, buflen, "X509 - Destination buffer is too small" ); if( use_ret == -(MBEDTLS_ERR_X509_FATAL_ERROR) ) - mbedtls_snprintf( buf, buflen, "X509 - A fatal error occured, eg the chain is too long or the vrfy callback failed" ); + mbedtls_snprintf( buf, buflen, "X509 - A fatal error occurred, eg the chain is too long or the vrfy callback failed" ); #endif /* MBEDTLS_X509_USE_C || MBEDTLS_X509_CREATE_C */ // END generated code diff --git a/thirdparty/mbedtls/library/havege.c b/thirdparty/mbedtls/library/havege.c index 54f897c6e7a..c139e1db03e 100644 --- a/thirdparty/mbedtls/library/havege.c +++ b/thirdparty/mbedtls/library/havege.c @@ -38,8 +38,19 @@ #include "mbedtls/timing.h" #include "mbedtls/platform_util.h" +#include #include +/* If int isn't capable of storing 2^32 distinct values, the code of this + * module may cause a processor trap or a miscalculation. If int is more + * than 32 bits, the code may not calculate the intended values. */ +#if INT_MIN + 1 != -0x7fffffff +#error "The HAVEGE module requires int to be exactly 32 bits, with INT_MIN = -2^31." +#endif +#if UINT_MAX != 0xffffffff +#error "The HAVEGE module requires unsigned to be exactly 32 bits." +#endif + /* ------------------------------------------------------------------------ * On average, one iteration accesses two 8-word blocks in the havege WALK * table, and generates 16 words in the RES array. @@ -54,7 +65,7 @@ * ------------------------------------------------------------------------ */ -#define SWAP(X,Y) { int *T = (X); (X) = (Y); (Y) = T; } +#define SWAP(X,Y) { unsigned *T = (X); (X) = (Y); (Y) = T; } #define TST1_ENTER if( PTEST & 1 ) { PTEST ^= 3; PTEST >>= 1; #define TST2_ENTER if( PTEST & 1 ) { PTEST ^= 3; PTEST >>= 1; @@ -77,7 +88,7 @@ PTX = (PT1 >> 18) & 7; \ PT1 &= 0x1FFF; \ PT2 &= 0x1FFF; \ - CLK = (int) mbedtls_timing_hardclock(); \ + CLK = (unsigned) mbedtls_timing_hardclock(); \ \ i = 0; \ A = &WALK[PT1 ]; RES[i++] ^= *A; \ @@ -100,7 +111,7 @@ \ IN = (*A >> (5)) ^ (*A << (27)) ^ CLK; \ *A = (*B >> (6)) ^ (*B << (26)) ^ CLK; \ - *B = IN; CLK = (int) mbedtls_timing_hardclock(); \ + *B = IN; CLK = (unsigned) mbedtls_timing_hardclock(); \ *C = (*C >> (7)) ^ (*C << (25)) ^ CLK; \ *D = (*D >> (8)) ^ (*D << (24)) ^ CLK; \ \ @@ -151,19 +162,20 @@ PT1 ^= (PT2 ^ 0x10) & 0x10; \ \ for( n++, i = 0; i < 16; i++ ) \ - hs->pool[n % MBEDTLS_HAVEGE_COLLECT_SIZE] ^= RES[i]; + POOL[n % MBEDTLS_HAVEGE_COLLECT_SIZE] ^= RES[i]; /* * Entropy gathering function */ static void havege_fill( mbedtls_havege_state *hs ) { - int i, n = 0; - int U1, U2, *A, *B, *C, *D; - int PT1, PT2, *WALK, RES[16]; - int PTX, PTY, CLK, PTEST, IN; + unsigned i, n = 0; + unsigned U1, U2, *A, *B, *C, *D; + unsigned PT1, PT2, *WALK, *POOL, RES[16]; + unsigned PTX, PTY, CLK, PTEST, IN; - WALK = hs->WALK; + WALK = (unsigned *) hs->WALK; + POOL = (unsigned *) hs->pool; PT1 = hs->PT1; PT2 = hs->PT2; diff --git a/thirdparty/mbedtls/library/hmac_drbg.c b/thirdparty/mbedtls/library/hmac_drbg.c index c50330e7d8b..50d88bd54b6 100644 --- a/thirdparty/mbedtls/library/hmac_drbg.c +++ b/thirdparty/mbedtls/library/hmac_drbg.c @@ -149,20 +149,32 @@ int mbedtls_hmac_drbg_seed_buf( mbedtls_hmac_drbg_context *ctx, } /* - * HMAC_DRBG reseeding: 10.1.2.4 (arabic) + 9.2 (Roman) + * Internal function used both for seeding and reseeding the DRBG. + * Comments starting with arabic numbers refer to section 10.1.2.4 + * of SP800-90A, while roman numbers refer to section 9.2. */ -int mbedtls_hmac_drbg_reseed( mbedtls_hmac_drbg_context *ctx, - const unsigned char *additional, size_t len ) +static int hmac_drbg_reseed_core( mbedtls_hmac_drbg_context *ctx, + const unsigned char *additional, size_t len, + int use_nonce ) { unsigned char seed[MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT]; - size_t seedlen; + size_t seedlen = 0; int ret; - /* III. Check input length */ - if( len > MBEDTLS_HMAC_DRBG_MAX_INPUT || - ctx->entropy_len + len > MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT ) { - return( MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG ); + size_t total_entropy_len; + + if( use_nonce == 0 ) + total_entropy_len = ctx->entropy_len; + else + total_entropy_len = ctx->entropy_len * 3 / 2; + + /* III. Check input length */ + if( len > MBEDTLS_HMAC_DRBG_MAX_INPUT || + total_entropy_len + len > MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT ) + { + return( MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG ); + } } memset( seed, 0, MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT ); @@ -170,9 +182,32 @@ int mbedtls_hmac_drbg_reseed( mbedtls_hmac_drbg_context *ctx, /* IV. Gather entropy_len bytes of entropy for the seed */ if( ( ret = ctx->f_entropy( ctx->p_entropy, seed, ctx->entropy_len ) ) != 0 ) + { return( MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED ); + } + seedlen += ctx->entropy_len; + + /* For initial seeding, allow adding of nonce generated + * from the entropy source. See Sect 8.6.7 in SP800-90A. */ + if( use_nonce ) + { + /* Note: We don't merge the two calls to f_entropy() in order + * to avoid requesting too much entropy from f_entropy() + * at once. Specifically, if the underlying digest is not + * SHA-1, 3 / 2 * entropy_len is at least 36 Bytes, which + * is larger than the maximum of 32 Bytes that our own + * entropy source implementation can emit in a single + * call in configurations disabling SHA-512. */ + if( ( ret = ctx->f_entropy( ctx->p_entropy, + seed + seedlen, + ctx->entropy_len / 2 ) ) != 0 ) + { + return( MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED ); + } + + seedlen += ctx->entropy_len / 2; + } - seedlen = ctx->entropy_len; /* 1. Concatenate entropy and additional data if any */ if( additional != NULL && len != 0 ) @@ -194,8 +229,20 @@ exit: return( ret ); } +/* + * HMAC_DRBG reseeding: 10.1.2.4 + 9.2 + */ +int mbedtls_hmac_drbg_reseed( mbedtls_hmac_drbg_context *ctx, + const unsigned char *additional, size_t len ) +{ + return( hmac_drbg_reseed_core( ctx, additional, len, 0 ) ); +} + /* * HMAC_DRBG initialisation (10.1.2.3 + 9.1) + * + * The nonce is not passed as a separate parameter but extracted + * from the entropy source as suggested in 8.6.7. */ int mbedtls_hmac_drbg_seed( mbedtls_hmac_drbg_context *ctx, const mbedtls_md_info_t * md_info, @@ -205,7 +252,7 @@ int mbedtls_hmac_drbg_seed( mbedtls_hmac_drbg_context *ctx, size_t len ) { int ret; - size_t entropy_len, md_size; + size_t md_size; if( ( ret = mbedtls_md_setup( &ctx->md_ctx, md_info, 1 ) ) != 0 ) return( ret ); @@ -233,20 +280,15 @@ int mbedtls_hmac_drbg_seed( mbedtls_hmac_drbg_context *ctx, * * (This also matches the sizes used in the NIST test vectors.) */ - entropy_len = md_size <= 20 ? 16 : /* 160-bits hash -> 128 bits */ - md_size <= 28 ? 24 : /* 224-bits hash -> 192 bits */ - 32; /* better (256+) -> 256 bits */ + ctx->entropy_len = md_size <= 20 ? 16 : /* 160-bits hash -> 128 bits */ + md_size <= 28 ? 24 : /* 224-bits hash -> 192 bits */ + 32; /* better (256+) -> 256 bits */ - /* - * For initialisation, use more entropy to emulate a nonce - * (Again, matches test vectors.) - */ - ctx->entropy_len = entropy_len * 3 / 2; - - if( ( ret = mbedtls_hmac_drbg_reseed( ctx, custom, len ) ) != 0 ) + if( ( ret = hmac_drbg_reseed_core( ctx, custom, len, + 1 /* add nonce */ ) ) != 0 ) + { return( ret ); - - ctx->entropy_len = entropy_len; + } return( 0 ); } diff --git a/thirdparty/mbedtls/library/net_sockets.c b/thirdparty/mbedtls/library/net_sockets.c index 816b1303dfa..5d538bfd562 100644 --- a/thirdparty/mbedtls/library/net_sockets.c +++ b/thirdparty/mbedtls/library/net_sockets.c @@ -284,7 +284,7 @@ static int net_would_block( const mbedtls_net_context *ctx ) int err = errno; /* - * Never return 'WOULD BLOCK' on a non-blocking socket + * Never return 'WOULD BLOCK' on a blocking socket */ if( ( fcntl( ctx->fd, F_GETFL ) & O_NONBLOCK ) != O_NONBLOCK ) { diff --git a/thirdparty/mbedtls/library/pkwrite.c b/thirdparty/mbedtls/library/pkwrite.c index 8d1da2f757f..03d14f2ff97 100644 --- a/thirdparty/mbedtls/library/pkwrite.c +++ b/thirdparty/mbedtls/library/pkwrite.c @@ -38,7 +38,9 @@ #include "mbedtls/rsa.h" #endif #if defined(MBEDTLS_ECP_C) +#include "mbedtls/bignum.h" #include "mbedtls/ecp.h" +#include "mbedtls/platform_util.h" #endif #if defined(MBEDTLS_ECDSA_C) #include "mbedtls/ecdsa.h" @@ -150,6 +152,26 @@ static int pk_write_ec_param( unsigned char **p, unsigned char *start, return( (int) len ); } + +/* + * privateKey OCTET STRING -- always of length ceil(log2(n)/8) + */ +static int pk_write_ec_private( unsigned char **p, unsigned char *start, + mbedtls_ecp_keypair *ec ) +{ + int ret; + size_t byte_length = ( ec->grp.pbits + 7 ) / 8; + unsigned char tmp[MBEDTLS_ECP_MAX_BYTES]; + + ret = mbedtls_mpi_write_binary( &ec->d, tmp, byte_length ); + if( ret != 0 ) + goto exit; + ret = mbedtls_asn1_write_octet_string( p, start, tmp, byte_length ); + +exit: + mbedtls_platform_zeroize( tmp, byte_length ); + return( ret ); +} #endif /* MBEDTLS_ECP_C */ int mbedtls_pk_write_pubkey( unsigned char **p, unsigned char *start, @@ -364,9 +386,8 @@ int mbedtls_pk_write_key_der( mbedtls_pk_context *key, unsigned char *buf, size_ MBEDTLS_ASN1_CONTEXT_SPECIFIC | MBEDTLS_ASN1_CONSTRUCTED | 0 ) ); len += par_len; - /* privateKey: write as MPI then fix tag */ - MBEDTLS_ASN1_CHK_ADD( len, mbedtls_asn1_write_mpi( &c, buf, &ec->d ) ); - *c = MBEDTLS_ASN1_OCTET_STRING; + /* privateKey */ + MBEDTLS_ASN1_CHK_ADD( len, pk_write_ec_private( &c, buf, ec ) ); /* version */ MBEDTLS_ASN1_CHK_ADD( len, mbedtls_asn1_write_int( &c, buf, 1 ) ); diff --git a/thirdparty/mbedtls/library/platform_util.c b/thirdparty/mbedtls/library/platform_util.c index 756e22679af..b1f745097ca 100644 --- a/thirdparty/mbedtls/library/platform_util.c +++ b/thirdparty/mbedtls/library/platform_util.c @@ -72,7 +72,10 @@ static void * (* const volatile memset_func)( void *, int, size_t ) = memset; void mbedtls_platform_zeroize( void *buf, size_t len ) { - memset_func( buf, 0, len ); + MBEDTLS_INTERNAL_VALIDATE( len == 0 || buf != NULL ); + + if( len > 0 ) + memset_func( buf, 0, len ); } #endif /* MBEDTLS_PLATFORM_ZEROIZE_ALT */ diff --git a/thirdparty/mbedtls/library/ssl_srv.c b/thirdparty/mbedtls/library/ssl_srv.c index bc77f802034..5825970c43c 100644 --- a/thirdparty/mbedtls/library/ssl_srv.c +++ b/thirdparty/mbedtls/library/ssl_srv.c @@ -1449,7 +1449,7 @@ read_record_header: */ /* - * Minimal length (with everything empty and extensions ommitted) is + * Minimal length (with everything empty and extensions omitted) is * 2 + 32 + 1 + 2 + 1 = 38 bytes. Check that first, so that we can * read at least up to session id length without worrying. */ diff --git a/thirdparty/mbedtls/library/ssl_tls.c b/thirdparty/mbedtls/library/ssl_tls.c index 38690fa6643..b8f35fec5d3 100644 --- a/thirdparty/mbedtls/library/ssl_tls.c +++ b/thirdparty/mbedtls/library/ssl_tls.c @@ -2606,7 +2606,7 @@ int mbedtls_ssl_fetch_input( mbedtls_ssl_context *ssl, size_t nb_want ) } /* - * A record can't be split accross datagrams. If we need to read but + * A record can't be split across datagrams. If we need to read but * are not at the beginning of a new record, the caller did something * wrong. */ @@ -9043,8 +9043,12 @@ static int ssl_preset_suiteb_hashes[] = { #if defined(MBEDTLS_ECP_C) static mbedtls_ecp_group_id ssl_preset_suiteb_curves[] = { +#if defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) MBEDTLS_ECP_DP_SECP256R1, +#endif +#if defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) MBEDTLS_ECP_DP_SECP384R1, +#endif MBEDTLS_ECP_DP_NONE }; #endif diff --git a/thirdparty/mbedtls/library/timing.c b/thirdparty/mbedtls/library/timing.c index 413d133fb6d..009516a6e3c 100644 --- a/thirdparty/mbedtls/library/timing.c +++ b/thirdparty/mbedtls/library/timing.c @@ -51,7 +51,6 @@ #if defined(_WIN32) && !defined(EFIX64) && !defined(EFI32) #include -#include #include struct _hr_time diff --git a/thirdparty/mbedtls/library/version_features.c b/thirdparty/mbedtls/library/version_features.c index 24143d052c3..a99ee808d69 100644 --- a/thirdparty/mbedtls/library/version_features.c +++ b/thirdparty/mbedtls/library/version_features.c @@ -87,6 +87,9 @@ static const char *features[] = { #if defined(MBEDTLS_CHECK_PARAMS) "MBEDTLS_CHECK_PARAMS", #endif /* MBEDTLS_CHECK_PARAMS */ +#if defined(MBEDTLS_CHECK_PARAMS_ASSERT) + "MBEDTLS_CHECK_PARAMS_ASSERT", +#endif /* MBEDTLS_CHECK_PARAMS_ASSERT */ #if defined(MBEDTLS_TIMING_ALT) "MBEDTLS_TIMING_ALT", #endif /* MBEDTLS_TIMING_ALT */ diff --git a/thirdparty/mbedtls/library/x509.c b/thirdparty/mbedtls/library/x509.c index a562df7ca3a..2e0b0e8f6cd 100644 --- a/thirdparty/mbedtls/library/x509.c +++ b/thirdparty/mbedtls/library/x509.c @@ -123,7 +123,7 @@ int mbedtls_x509_get_alg_null( unsigned char **p, const unsigned char *end, } /* - * Parse an algorithm identifier with (optional) paramaters + * Parse an algorithm identifier with (optional) parameters */ int mbedtls_x509_get_alg( unsigned char **p, const unsigned char *end, mbedtls_x509_buf *alg, mbedtls_x509_buf *params ) diff --git a/thirdparty/mbedtls/library/x509_crt.c b/thirdparty/mbedtls/library/x509_crt.c index 97e1d72e3c7..a3697f13f9c 100644 --- a/thirdparty/mbedtls/library/x509_crt.c +++ b/thirdparty/mbedtls/library/x509_crt.c @@ -65,6 +65,19 @@ #if defined(_WIN32) && !defined(EFIX64) && !defined(EFI32) #include +#if defined(_MSC_VER) && _MSC_VER <= 1600 +/* Visual Studio 2010 and earlier issue a warning when both and + * are included, as they redefine a number of _MAX constants. + * These constants are guaranteed to be the same, though, so we suppress the + * warning when including intsafe.h. + */ +#pragma warning( push ) +#pragma warning( disable : 4005 ) +#endif +#include +#if defined(_MSC_VER) && _MSC_VER <= 1600 +#pragma warning( pop ) +#endif #else #include #endif @@ -1277,6 +1290,7 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) char filename[MAX_PATH]; char *p; size_t len = strlen( path ); + int lengthAsInt = 0; WIN32_FIND_DATAW file_data; HANDLE hFind; @@ -1291,7 +1305,18 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) p = filename + len; filename[len++] = '*'; - w_ret = MultiByteToWideChar( CP_ACP, 0, filename, (int)len, szDir, + if ( FAILED ( SizeTToInt( len, &lengthAsInt ) ) ) + return( MBEDTLS_ERR_X509_FILE_IO_ERROR ); + + /* + * Note this function uses the code page CP_ACP, and assumes the incoming + * string is encoded in ANSI, before translating it into Unicode. If the + * incoming string were changed to be UTF-8, then the length check needs to + * change to check the number of characters, not the number of bytes, in the + * incoming string are less than MAX_PATH to avoid a buffer overrun with + * MultiByteToWideChar(). + */ + w_ret = MultiByteToWideChar( CP_ACP, 0, filename, lengthAsInt, szDir, MAX_PATH - 3 ); if( w_ret == 0 ) return( MBEDTLS_ERR_X509_BAD_INPUT_DATA ); @@ -1308,8 +1333,11 @@ int mbedtls_x509_crt_parse_path( mbedtls_x509_crt *chain, const char *path ) if( file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) continue; + if ( FAILED( SizeTToInt( wcslen( file_data.cFileName ), &lengthAsInt ) ) ) + return( MBEDTLS_ERR_X509_FILE_IO_ERROR ); + w_ret = WideCharToMultiByte( CP_ACP, 0, file_data.cFileName, - lstrlenW( file_data.cFileName ), + lengthAsInt, p, (int) len - 1, NULL, NULL ); if( w_ret == 0 ) @@ -2087,15 +2115,13 @@ check_signature: continue; } + *r_parent = parent; + *r_signature_is_good = signature_is_good; + break; } - if( parent != NULL ) - { - *r_parent = parent; - *r_signature_is_good = signature_is_good; - } - else + if( parent == NULL ) { *r_parent = fallback_parent; *r_signature_is_good = fallback_signature_is_good; @@ -2236,7 +2262,7 @@ static int x509_crt_check_ee_locally_trusted( * Tests for (aspects of) this function should include at least: * - trusted EE * - EE -> trusted root - * - EE -> intermedate CA -> trusted root + * - EE -> intermediate CA -> trusted root * - if relevant: EE untrusted * - if relevant: EE -> intermediate, untrusted * with the aspect under test checked at each relevant level (EE, int, root). diff --git a/thirdparty/mbedtls/library/x509write_crt.c b/thirdparty/mbedtls/library/x509write_crt.c index 10497e752bb..61d7ba44a03 100644 --- a/thirdparty/mbedtls/library/x509write_crt.c +++ b/thirdparty/mbedtls/library/x509write_crt.c @@ -45,6 +45,16 @@ #include "mbedtls/pem.h" #endif /* MBEDTLS_PEM_WRITE_C */ +/* + * For the currently used signature algorithms the buffer to store any signature + * must be at least of size MAX(MBEDTLS_ECDSA_MAX_LEN, MBEDTLS_MPI_MAX_SIZE) + */ +#if MBEDTLS_ECDSA_MAX_LEN > MBEDTLS_MPI_MAX_SIZE +#define SIGNATURE_MAX_SIZE MBEDTLS_ECDSA_MAX_LEN +#else +#define SIGNATURE_MAX_SIZE MBEDTLS_MPI_MAX_SIZE +#endif + void mbedtls_x509write_crt_init( mbedtls_x509write_cert *ctx ) { memset( ctx, 0, sizeof( mbedtls_x509write_cert ) ); @@ -334,7 +344,7 @@ int mbedtls_x509write_crt_der( mbedtls_x509write_cert *ctx, unsigned char *buf, size_t sig_oid_len = 0; unsigned char *c, *c2; unsigned char hash[64]; - unsigned char sig[MBEDTLS_MPI_MAX_SIZE]; + unsigned char sig[SIGNATURE_MAX_SIZE]; unsigned char tmp_buf[2048]; size_t sub_len = 0, pub_len = 0, sig_and_oid_len = 0, sig_len; size_t len = 0; diff --git a/thirdparty/mbedtls/library/x509write_csr.c b/thirdparty/mbedtls/library/x509write_csr.c index d70ba0ed927..b65a11c6aa7 100644 --- a/thirdparty/mbedtls/library/x509write_csr.c +++ b/thirdparty/mbedtls/library/x509write_csr.c @@ -44,6 +44,16 @@ #include "mbedtls/pem.h" #endif +/* + * For the currently used signature algorithms the buffer to store any signature + * must be at least of size MAX(MBEDTLS_ECDSA_MAX_LEN, MBEDTLS_MPI_MAX_SIZE) + */ +#if MBEDTLS_ECDSA_MAX_LEN > MBEDTLS_MPI_MAX_SIZE +#define SIGNATURE_MAX_SIZE MBEDTLS_ECDSA_MAX_LEN +#else +#define SIGNATURE_MAX_SIZE MBEDTLS_MPI_MAX_SIZE +#endif + void mbedtls_x509write_csr_init( mbedtls_x509write_csr *ctx ) { memset( ctx, 0, sizeof( mbedtls_x509write_csr ) ); @@ -159,7 +169,7 @@ int mbedtls_x509write_csr_der( mbedtls_x509write_csr *ctx, unsigned char *buf, s size_t sig_oid_len = 0; unsigned char *c, *c2; unsigned char hash[64]; - unsigned char sig[MBEDTLS_MPI_MAX_SIZE]; + unsigned char sig[SIGNATURE_MAX_SIZE]; unsigned char tmp_buf[2048]; size_t pub_len = 0, sig_and_oid_len = 0, sig_len; size_t len = 0; diff --git a/thirdparty/mbedtls/1453.diff b/thirdparty/mbedtls/patches/1453.diff similarity index 100% rename from thirdparty/mbedtls/1453.diff rename to thirdparty/mbedtls/patches/1453.diff diff --git a/thirdparty/mbedtls/padlock.diff b/thirdparty/mbedtls/patches/padlock.diff similarity index 100% rename from thirdparty/mbedtls/padlock.diff rename to thirdparty/mbedtls/patches/padlock.diff From 6b972f178a74b27f9d529a9e0855dff6bb5f2a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 09:06:31 +0100 Subject: [PATCH 019/113] miniupnpc: Sync with upstream master (0ab1d67) (cherry picked from commit e2fbfb1dedaf7abead06db858ea83ca2f7278c13) --- thirdparty/README.md | 9 ++++---- thirdparty/miniupnpc/LICENSE | 1 - .../miniupnpc/miniupnpc/connecthostport.c | 6 ++++- thirdparty/miniupnpc/miniupnpc/miniupnpc.c | 9 ++++---- thirdparty/miniupnpc/miniupnpc/upnpc.c | 23 +++++++++++-------- thirdparty/miniupnpc/miniupnpc/upnperrors.c | 9 ++++---- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index 0c29563c889..ce36bba70e7 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -302,12 +302,13 @@ File extracted from upstream release tarball (`-apache.tgz` variant): ## miniupnpc - Upstream: https://github.com/miniupnp/miniupnp/tree/master/miniupnpc -- Version: git (3cf6efa, 2019) +- Version: git (0ab1d67, 2019) - License: BSD-3-Clause -Extract only the `miniupnpc` folder inside `thirdparty/miniupnpc`. -Exclude all non `.c` and `.h` files, plus all files beginning with `test` -`minihttptestserver.c` and `wingenminiupnpcstrings.c`. +Files extracted from upstream source: + +- All `*.c` and `*.h` files from `miniupnpc` to `thirdparty/miniupnpc/miniupnpc` +- Remove `test*`, `minihttptestserver.c` and `wingenminiupnpcstrings.c` The only modified file is miniupnpcstrings.h, which was created for Godot (it is usually autogenerated by cmake). diff --git a/thirdparty/miniupnpc/LICENSE b/thirdparty/miniupnpc/LICENSE index 39e0345f8a7..1460310752d 100644 --- a/thirdparty/miniupnpc/LICENSE +++ b/thirdparty/miniupnpc/LICENSE @@ -24,4 +24,3 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/thirdparty/miniupnpc/miniupnpc/connecthostport.c b/thirdparty/miniupnpc/miniupnpc/connecthostport.c index a59dc824372..f3982e1a77d 100644 --- a/thirdparty/miniupnpc/miniupnpc/connecthostport.c +++ b/thirdparty/miniupnpc/miniupnpc/connecthostport.c @@ -1,4 +1,4 @@ -/* $Id: connecthostport.c,v 1.21 2019/04/23 12:11:08 nanard Exp $ */ +/* $Id: connecthostport.c,v 1.22 2019/10/13 17:22:08 nanard Exp $ */ /* vim: tabstop=4 shiftwidth=4 noexpandtab * Project : miniupnp * Author : Thomas Bernard @@ -195,6 +195,10 @@ SOCKET connecthostport(const char * host, unsigned short port, { if(!ISINVALID(s)) closesocket(s); +#ifdef DEBUG + printf("ai_family=%d ai_socktype=%d ai_protocol=%d (PF_INET=%d, PF_INET6=%d)\n", + p->ai_family, p->ai_socktype, p->ai_protocol, PF_INET, PF_INET6); +#endif s = socket(p->ai_family, p->ai_socktype, p->ai_protocol); if(ISINVALID(s)) continue; diff --git a/thirdparty/miniupnpc/miniupnpc/miniupnpc.c b/thirdparty/miniupnpc/miniupnpc/miniupnpc.c index 3181d10eb63..95ab6cf56b7 100644 --- a/thirdparty/miniupnpc/miniupnpc/miniupnpc.c +++ b/thirdparty/miniupnpc/miniupnpc/miniupnpc.c @@ -564,6 +564,7 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, char * lanaddr, int lanaddrlen) { struct xml_desc { + char lanaddr[40]; char * xml; int size; int is_igd; @@ -573,7 +574,6 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, int i; int state = -1; /* state 1 : IGD connected. State 2 : IGD. State 3 : anything */ char extIpAddr[16]; - char myLanAddr[40]; int status_code = -1; if(!devlist) @@ -596,7 +596,7 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, /* we should choose an internet gateway device. * with st == urn:schemas-upnp-org:device:InternetGatewayDevice:1 */ desc[i].xml = miniwget_getaddr(dev->descURL, &(desc[i].size), - myLanAddr, sizeof(myLanAddr), + desc[i].lanaddr, sizeof(desc[i].lanaddr), dev->scope_id, &status_code); #ifdef DEBUG if(!desc[i].xml) @@ -613,8 +613,6 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, "urn:schemas-upnp-org:service:WANCommonInterfaceConfig:")) { desc[i].is_igd = 1; - if(lanaddr) - strncpy(lanaddr, myLanAddr, lanaddrlen); } } } @@ -680,6 +678,8 @@ UPNP_GetValidIGD(struct UPNPDev * devlist, } state = 0; free_and_return: + if (lanaddr != NULL && state >= 1 && state <= 3 && i < ndev) + strncpy(lanaddr, desc[i].lanaddr, lanaddrlen); for(i = 0; i < ndev; i++) free(desc[i].xml); free(desc); @@ -713,4 +713,3 @@ UPNP_GetIGDFromUrl(const char * rootdescurl, return 0; } } - diff --git a/thirdparty/miniupnpc/miniupnpc/upnpc.c b/thirdparty/miniupnpc/miniupnpc/upnpc.c index 674c89beb0d..4325658bee8 100644 --- a/thirdparty/miniupnpc/miniupnpc/upnpc.c +++ b/thirdparty/miniupnpc/miniupnpc/upnpc.c @@ -250,6 +250,7 @@ static int SetRedirectAndTest(struct UPNPUrls * urls, const char * eport, const char * proto, const char * leaseDuration, + const char * remoteHost, const char * description, int addAny) { @@ -283,7 +284,7 @@ static int SetRedirectAndTest(struct UPNPUrls * urls, if (addAny) { r = UPNP_AddAnyPortMapping(urls->controlURL, data->first.servicetype, eport, iport, iaddr, description, - proto, 0, leaseDuration, reservedPort); + proto, remoteHost, leaseDuration, reservedPort); if(r==UPNPCOMMAND_SUCCESS) eport = reservedPort; else @@ -292,7 +293,7 @@ static int SetRedirectAndTest(struct UPNPUrls * urls, } else { r = UPNP_AddPortMapping(urls->controlURL, data->first.servicetype, eport, iport, iaddr, description, - proto, 0, leaseDuration); + proto, remoteHost, leaseDuration); if(r!=UPNPCOMMAND_SUCCESS) { printf("AddPortMapping(%s, %s, %s) failed with code %d (%s)\n", eport, iport, iaddr, r, strupnperror(r)); @@ -302,7 +303,7 @@ static int SetRedirectAndTest(struct UPNPUrls * urls, r = UPNP_GetSpecificPortMappingEntry(urls->controlURL, data->first.servicetype, - eport, proto, NULL/*remoteHost*/, + eport, proto, remoteHost, intClient, intPort, NULL/*desc*/, NULL/*enabled*/, duration); if(r!=UPNPCOMMAND_SUCCESS) { @@ -642,12 +643,12 @@ int main(int argc, char ** argv) || (command == 'U' && commandargc<2) || (command == 'D' && commandargc<1)) { - fprintf(stderr, "Usage :\t%s [options] -a ip port external_port protocol [duration]\n\t\tAdd port redirection\n", argv[0]); - fprintf(stderr, " \t%s [options] -d external_port protocol \n\t\tDelete port redirection\n", argv[0]); + fprintf(stderr, "Usage :\t%s [options] -a ip port external_port protocol [duration] [remote host]\n\t\tAdd port redirection\n", argv[0]); + fprintf(stderr, " \t%s [options] -d external_port protocol [remote host]\n\t\tDelete port redirection\n", argv[0]); fprintf(stderr, " \t%s [options] -s\n\t\tGet Connection status\n", argv[0]); fprintf(stderr, " \t%s [options] -l\n\t\tList redirections\n", argv[0]); fprintf(stderr, " \t%s [options] -L\n\t\tList redirections (using GetListOfPortMappings (for IGD:2 only)\n", argv[0]); - fprintf(stderr, " \t%s [options] -n ip port external_port protocol [duration]\n\t\tAdd (any) port redirection allowing IGD to use alternative external_port (for IGD:2 only)\n", argv[0]); + fprintf(stderr, " \t%s [options] -n ip port external_port protocol [duration] [remote host]\n\t\tAdd (any) port redirection allowing IGD to use alternative external_port (for IGD:2 only)\n", argv[0]); fprintf(stderr, " \t%s [options] -N external_port_start external_port_end protocol [manage]\n\t\tDelete range of port redirections (for IGD:2 only)\n", argv[0]); fprintf(stderr, " \t%s [options] -r port1 [external_port1] protocol1 [port2 [external_port2] protocol2] [...]\n\t\tAdd all redirections to the current host\n", argv[0]); fprintf(stderr, " \t%s [options] -A remote_ip remote_port internal_ip internal_port protocol lease_time\n\t\tAdd Pinhole (for IGD:2 only)\n", argv[0]); @@ -734,7 +735,8 @@ int main(int argc, char ** argv) if (SetRedirectAndTest(&urls, &data, commandargv[0], commandargv[1], commandargv[2], commandargv[3], - (commandargc > 4)?commandargv[4]:"0", + (commandargc > 4)&is_int(commandargv[4])?commandargv[4]:"0", + (commandargc > 4)&!is_int(commandargv[4])?commandargv[4]:(commandargc > 5)?commandargv[5]:NULL, description, 0) < 0) retcode = 2; break; @@ -747,7 +749,8 @@ int main(int argc, char ** argv) if (SetRedirectAndTest(&urls, &data, commandargv[0], commandargv[1], commandargv[2], commandargv[3], - (commandargc > 4)?commandargv[4]:"0", + (commandargc > 4)&is_int(commandargv[4])?commandargv[4]:"0", + (commandargc > 4)&!is_int(commandargv[4])?commandargv[4]:(commandargc > 5)?commandargv[5]:NULL, description, 1) < 0) retcode = 2; break; @@ -775,7 +778,7 @@ int main(int argc, char ** argv) /* 2nd parameter is an integer : */ if (SetRedirectAndTest(&urls, &data, lanaddr, commandargv[i], - commandargv[i+1], commandargv[i+2], "0", + commandargv[i+1], commandargv[i+2], "0", NULL, description, 0) < 0) retcode = 2; i+=3; /* 3 parameters parsed */ @@ -783,7 +786,7 @@ int main(int argc, char ** argv) /* 2nd parameter not an integer : */ if (SetRedirectAndTest(&urls, &data, lanaddr, commandargv[i], - commandargv[i], commandargv[i+1], "0", + commandargv[i], commandargv[i+1], "0", NULL, description, 0) < 0) retcode = 2; i+=2; /* 2 parameters parsed */ diff --git a/thirdparty/miniupnpc/miniupnpc/upnperrors.c b/thirdparty/miniupnpc/miniupnpc/upnperrors.c index 650af42557a..4496e8622c3 100644 --- a/thirdparty/miniupnpc/miniupnpc/upnperrors.c +++ b/thirdparty/miniupnpc/miniupnpc/upnperrors.c @@ -1,9 +1,10 @@ -/* $Id: upnperrors.c,v 1.9 2019/06/25 21:15:46 nanard Exp $ */ -/* Project : miniupnp +/* $Id: upnperrors.c,v 1.10 2019/08/24 08:49:53 nanard Exp $ */ +/* vim: tabstop=4 shiftwidth=4 noexpandtab + * Project : miniupnp * Author : Thomas BERNARD * copyright (c) 2007-2019 Thomas Bernard * All Right reserved. - * http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/ + * http://miniupnp.free.fr/ or https://miniupnp.tuxfamily.org/ * This software is subjet to the conditions detailed in the * provided LICENCE file. */ #include @@ -71,7 +72,7 @@ const char * strupnperror(int err) s = "ProtocolWildcardingNotAllowed"; break; case 708: - s = "WildcardNotPermittedInSrcIP"; + s = "InvalidLayer2Address"; break; case 709: s = "NoPacketSent"; From 908c8862ab8d737fa0ce4313657fcb85e2f6c112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 09:12:11 +0100 Subject: [PATCH 020/113] nanosvg: Sync with upstream c1f6e20 Now formally unmaintained, so probably no further sync to expect from this repository. (cherry picked from commit 097b746fddee5550acb3f0fab5887aeba7c8038e) --- thirdparty/README.md | 2 +- thirdparty/nanosvg/nanosvg.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index ce36bba70e7..45db78ac6ca 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -422,7 +422,7 @@ Collection of single-file libraries used in Godot components. ## nanosvg - Upstream: https://github.com/memononen/nanosvg -- Version: git (c1f6e20, 2018) +- Version: git (25241c5, 2019) - License: zlib Files extracted from the upstream source: diff --git a/thirdparty/nanosvg/nanosvg.h b/thirdparty/nanosvg/nanosvg.h index 8c8b061cd16..e5f69006144 100644 --- a/thirdparty/nanosvg/nanosvg.h +++ b/thirdparty/nanosvg/nanosvg.h @@ -1102,7 +1102,7 @@ static double nsvg__atof(const char* s) // Parse integer part if (nsvg__isdigit(*cur)) { // Parse digit sequence - intPart = (double)strtoll(cur, &end, 10); + intPart = strtoll(cur, &end, 10); if (cur != end) { res = (double)intPart; hasIntPart = 1; @@ -1130,7 +1130,7 @@ static double nsvg__atof(const char* s) // Parse optional exponent if (*cur == 'e' || *cur == 'E') { - int expPart = 0; + long expPart = 0; cur++; // skip 'E' expPart = strtol(cur, &end, 10); // Parse digit sequence with sign if (cur != end) { @@ -1168,7 +1168,7 @@ static const char* nsvg__parseNumber(const char* s, char* it, const int size) } } // exponent - if (*s == 'e' || *s == 'E') { + if ((*s == 'e' || *s == 'E') && (s[1] != 'm' && s[1] != 'x')) { if (i < last) it[i++] = *s; s++; if (*s == '-' || *s == '+') { From 24465c5e254443333121d3fd2cebf6dd888a7fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 09:17:05 +0100 Subject: [PATCH 021/113] tinyexr: Sync with upstream 656bb61 (cherry picked from commit 6a2c2e506242a7da3c612ed9a148a4eec241f20d) --- thirdparty/README.md | 2 +- thirdparty/tinyexr/tinyexr.h | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/thirdparty/README.md b/thirdparty/README.md index 45db78ac6ca..810d8b3dd23 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -507,7 +507,7 @@ comments and a patch is provided in the squish/ folder. ## tinyexr - Upstream: https://github.com/syoyo/tinyexr -- Version: git (65f9859, 2018) +- Version: git (656bb61, 2019) - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/tinyexr/tinyexr.h b/thirdparty/tinyexr/tinyexr.h index f22163738fe..bfc52b51a57 100644 --- a/thirdparty/tinyexr/tinyexr.h +++ b/thirdparty/tinyexr/tinyexr.h @@ -111,13 +111,13 @@ extern "C" { #define TINYEXR_ERROR_INVALID_ARGUMENT (-3) #define TINYEXR_ERROR_INVALID_DATA (-4) #define TINYEXR_ERROR_INVALID_FILE (-5) -#define TINYEXR_ERROR_INVALID_PARAMETER (-5) -#define TINYEXR_ERROR_CANT_OPEN_FILE (-6) -#define TINYEXR_ERROR_UNSUPPORTED_FORMAT (-7) -#define TINYEXR_ERROR_INVALID_HEADER (-8) -#define TINYEXR_ERROR_UNSUPPORTED_FEATURE (-9) -#define TINYEXR_ERROR_CANT_WRITE_FILE (-10) -#define TINYEXR_ERROR_SERIALZATION_FAILED (-11) +#define TINYEXR_ERROR_INVALID_PARAMETER (-6) +#define TINYEXR_ERROR_CANT_OPEN_FILE (-7) +#define TINYEXR_ERROR_UNSUPPORTED_FORMAT (-8) +#define TINYEXR_ERROR_INVALID_HEADER (-9) +#define TINYEXR_ERROR_UNSUPPORTED_FEATURE (-10) +#define TINYEXR_ERROR_CANT_WRITE_FILE (-11) +#define TINYEXR_ERROR_SERIALZATION_FAILED (-12) // @note { OpenEXR file format: http://www.openexr.com/openexrfilelayout.pdf } @@ -554,6 +554,10 @@ namespace miniz { #pragma clang diagnostic ignored "-Wtautological-constant-compare" #endif +#if __has_warning("-Wextra-semi-stmt") +#pragma clang diagnostic ignored "-Wextra-semi-stmt" +#endif + #endif /* miniz.c v1.15 - public domain deflate/inflate, zlib-subset, ZIP @@ -7625,6 +7629,9 @@ static bool DecompressZip(unsigned char *dst, #ifdef __clang__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wsign-conversion" +#if __has_warning("-Wextra-semi-stmt") +#pragma clang diagnostic ignored "-Wextra-semi-stmt" +#endif #endif #ifdef _MSC_VER @@ -7886,6 +7893,10 @@ static bool DecompressRle(unsigned char *dst, #pragma clang diagnostic ignored "-Wcast-qual" #endif +#if __has_warning("-Wextra-semi-stmt") +#pragma clang diagnostic ignored "-Wextra-semi-stmt" +#endif + #endif // From a5f4f1551046ab90d0aff4dcdd9eacb495d2ee82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Mon, 11 Nov 2019 15:14:37 +0100 Subject: [PATCH 022/113] AudioStreamOGGVorbis: Expose loop and loop_offset as properties Fixes #11231. (cherry picked from commit 51d1be51561d1e698307a2a28bb5f53ea84d4f61) --- modules/stb_vorbis/audio_stream_ogg_vorbis.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp b/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp index 292ac5e97e6..4e9f2285e95 100644 --- a/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp +++ b/modules/stb_vorbis/audio_stream_ogg_vorbis.cpp @@ -266,8 +266,8 @@ void AudioStreamOGGVorbis::_bind_methods() { ClassDB::bind_method(D_METHOD("get_loop_offset"), &AudioStreamOGGVorbis::get_loop_offset); ADD_PROPERTY(PropertyInfo(Variant::POOL_BYTE_ARRAY, "data", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR), "set_data", "get_data"); - ADD_PROPERTY(PropertyInfo(Variant::BOOL, "loop", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR), "set_loop", "has_loop"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "loop_offset", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR), "set_loop_offset", "get_loop_offset"); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "loop"), "set_loop", "has_loop"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "loop_offset"), "set_loop_offset", "get_loop_offset"); } AudioStreamOGGVorbis::AudioStreamOGGVorbis() { From 39e76014d4a4c8c344569a05b7f975836af3974c Mon Sep 17 00:00:00 2001 From: Vixelz Date: Thu, 21 Feb 2019 14:23:48 +0000 Subject: [PATCH 023/113] Include global class resources in Resource property inspector Much like how script defined global classes can be created from the "New Resource..." button in the FileSystem panel, this allows the creation of script defined resources to be embedded. (cherry picked from commit 03f7a724ed8c1fc1601510338a349e72fe263e8c) --- editor/editor_properties.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/editor/editor_properties.cpp b/editor/editor_properties.cpp index c13a4a5a5e5..0e8cd955b56 100644 --- a/editor/editor_properties.cpp +++ b/editor/editor_properties.cpp @@ -2289,6 +2289,16 @@ void EditorPropertyResource::_update_menu_items() { E = E->next(); } + List global_classes; + ScriptServer::get_global_class_list(&global_classes); + E = global_classes.front(); + while (E) { + if (EditorNode::get_editor_data().script_class_is_parent(E->get(), base_type)) { + valid_inheritors.insert(E->get()); + } + E = E->next(); + } + for (Set::Element *F = valid_inheritors.front(); F; F = F->next()) { String t = F->get(); @@ -2305,7 +2315,7 @@ void EditorPropertyResource::_update_menu_items() { } } - if (!is_custom_resource && !ClassDB::can_instance(t)) + if (!is_custom_resource && !(ScriptServer::is_global_class(t) || ClassDB::can_instance(t))) continue; inheritors_array.push_back(t); From 7d84035e13f7445416671f35146c49c1e6f73e3c Mon Sep 17 00:00:00 2001 From: will Date: Sun, 3 Mar 2019 15:24:58 -0600 Subject: [PATCH 024/113] New shortcuts for the editor menu items (cherry picked from commit 7a880029929aa09cf5e9b281b50043df5bb17aa2) --- editor/editor_node.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index fef967508df..edcd7d64781 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -5440,7 +5440,7 @@ EditorNode::EditorNode() { p->add_shortcut(ED_SHORTCUT("editor/undo", TTR("Undo"), KEY_MASK_CMD + KEY_Z), EDIT_UNDO, true); p->add_shortcut(ED_SHORTCUT("editor/redo", TTR("Redo"), KEY_MASK_SHIFT + KEY_MASK_CMD + KEY_Z), EDIT_REDO, true); p->add_separator(); - p->add_item(TTR("Revert Scene"), EDIT_REVERT); + p->add_shortcut(ED_SHORTCUT("editor/revert_scene", TTR("Revert Scene")), EDIT_REVERT); recent_scenes = memnew(PopupMenu); recent_scenes->set_name("RecentScenes"); @@ -5460,10 +5460,10 @@ EditorNode::EditorNode() { p = project_menu->get_popup(); p->set_hide_on_window_lose_focus(true); - p->add_item(TTR("Project Settings"), RUN_SETTINGS); + p->add_shortcut(ED_SHORTCUT("editor/project_settings", TTR("Project Settings")), RUN_SETTINGS); p->add_separator(); p->connect("id_pressed", this, "_menu_option"); - p->add_item(TTR("Export"), FILE_EXPORT_PROJECT); + p->add_shortcut(ED_SHORTCUT("editor/export", TTR("Export")), FILE_EXPORT_PROJECT); plugin_config_dialog = memnew(PluginConfigDialog); plugin_config_dialog->connect("plugin_ready", this, "_on_plugin_ready"); @@ -5474,10 +5474,10 @@ EditorNode::EditorNode() { tool_menu->connect("index_pressed", this, "_tool_menu_option"); p->add_child(tool_menu); p->add_submenu_item(TTR("Tools"), "Tools"); - tool_menu->add_item(TTR("Orphan Resource Explorer"), TOOLS_ORPHAN_RESOURCES); + tool_menu->add_shortcut(ED_SHORTCUT("editor/orphan_resource_explorer", TTR("Orphan Resource Explorer")), TOOLS_ORPHAN_RESOURCES); p->add_separator(); - p->add_item(TTR("Open Project Data Folder"), RUN_PROJECT_DATA_FOLDER); + p->add_shortcut(ED_SHORTCUT("editor/open_project_data_folder", TTR("Open Project Data Folder")), RUN_PROJECT_DATA_FOLDER); p->add_separator(); #ifdef OSX_ENABLED @@ -5501,21 +5501,21 @@ EditorNode::EditorNode() { p = debug_menu->get_popup(); p->set_hide_on_window_lose_focus(true); p->set_hide_on_item_selection(false); - p->add_check_item(TTR("Deploy with Remote Debug"), RUN_DEPLOY_REMOTE_DEBUG); + p->add_check_shortcut(ED_SHORTCUT("editor/deploy_with_remote_debug", TTR("Deploy with Remote Debug")), RUN_DEPLOY_REMOTE_DEBUG); p->set_item_tooltip(p->get_item_count() - 1, TTR("When exporting or deploying, the resulting executable will attempt to connect to the IP of this computer in order to be debugged.")); - p->add_check_item(TTR("Small Deploy with Network FS"), RUN_FILE_SERVER); + p->add_check_shortcut(ED_SHORTCUT("editor/small_deploy_with_network_fs", TTR("Small Deploy with Network FS")), RUN_FILE_SERVER); p->set_item_tooltip(p->get_item_count() - 1, TTR("When this option is enabled, export or deploy will produce a minimal executable.\nThe filesystem will be provided from the project by the editor over the network.\nOn Android, deploy will use the USB cable for faster performance. This option speeds up testing for games with a large footprint.")); p->add_separator(); - p->add_check_item(TTR("Visible Collision Shapes"), RUN_DEBUG_COLLISONS); + p->add_check_shortcut(ED_SHORTCUT("editor/visible_collision_shapes", TTR("Visible Collision Shapes")), RUN_DEBUG_COLLISONS); p->set_item_tooltip(p->get_item_count() - 1, TTR("Collision shapes and raycast nodes (for 2D and 3D) will be visible on the running game if this option is turned on.")); - p->add_check_item(TTR("Visible Navigation"), RUN_DEBUG_NAVIGATION); + p->add_check_shortcut(ED_SHORTCUT("editor/visible_navigation", TTR("Visible Navigation")), RUN_DEBUG_NAVIGATION); p->set_item_tooltip(p->get_item_count() - 1, TTR("Navigation meshes and polygons will be visible on the running game if this option is turned on.")); p->add_separator(); //those are now on by default, since they are harmless - p->add_check_item(TTR("Sync Scene Changes"), RUN_LIVE_DEBUG); + p->add_check_shortcut(ED_SHORTCUT("editor/sync_scene_changes", TTR("Sync Scene Changes")), RUN_LIVE_DEBUG); p->set_item_tooltip(p->get_item_count() - 1, TTR("When this option is turned on, any changes made to the scene in the editor will be replicated in the running game.\nWhen used remotely on a device, this is more efficient with network filesystem.")); p->set_item_checked(p->get_item_count() - 1, true); - p->add_check_item(TTR("Sync Script Changes"), RUN_RELOAD_SCRIPTS); + p->add_check_shortcut(ED_SHORTCUT("editor/sync_script_changes", TTR("Sync Script Changes")), RUN_RELOAD_SCRIPTS); p->set_item_tooltip(p->get_item_count() - 1, TTR("When this option is turned on, any script that is saved will be reloaded on the running game.\nWhen used remotely on a device, this is more efficient with network filesystem.")); p->set_item_checked(p->get_item_count() - 1, true); p->connect("id_pressed", this, "_menu_option"); @@ -5531,7 +5531,7 @@ EditorNode::EditorNode() { p = settings_menu->get_popup(); p->set_hide_on_window_lose_focus(true); - p->add_item(TTR("Editor Settings"), SETTINGS_PREFERENCES); + p->add_shortcut(ED_SHORTCUT("editor/editor_settings", TTR("Editor Settings")), SETTINGS_PREFERENCES); p->add_separator(); editor_layouts = memnew(PopupMenu); @@ -5571,12 +5571,12 @@ EditorNode::EditorNode() { p->connect("id_pressed", this, "_menu_option"); p->add_icon_shortcut(gui_base->get_icon("HelpSearch", "EditorIcons"), ED_SHORTCUT("editor/editor_help", TTR("Search"), KEY_F4), HELP_SEARCH); p->add_separator(); - p->add_icon_item(gui_base->get_icon("Instance", "EditorIcons"), TTR("Online Docs"), HELP_DOCS); - p->add_icon_item(gui_base->get_icon("Instance", "EditorIcons"), TTR("Q&A"), HELP_QA); - p->add_icon_item(gui_base->get_icon("Instance", "EditorIcons"), TTR("Issue Tracker"), HELP_ISSUES); - p->add_icon_item(gui_base->get_icon("Instance", "EditorIcons"), TTR("Community"), HELP_COMMUNITY); + p->add_icon_shortcut(gui_base->get_icon("Instance", "EditorIcons"), ED_SHORTCUT("editor/online_docs", TTR("Online Docs")), HELP_DOCS); + p->add_icon_shortcut(gui_base->get_icon("Instance", "EditorIcons"), ED_SHORTCUT("editor/q&a", TTR("Q&A")), HELP_QA); + p->add_icon_shortcut(gui_base->get_icon("Instance", "EditorIcons"), ED_SHORTCUT("editor/issue_tracker", TTR("Issue Tracker")), HELP_ISSUES); + p->add_icon_shortcut(gui_base->get_icon("Instance", "EditorIcons"), ED_SHORTCUT("editor/community", TTR("Community")), HELP_COMMUNITY); p->add_separator(); - p->add_icon_item(gui_base->get_icon("Godot", "EditorIcons"), TTR("About"), HELP_ABOUT); + p->add_icon_shortcut(gui_base->get_icon("Godot", "EditorIcons"), ED_SHORTCUT("editor/about", TTR("About")), HELP_ABOUT); HBoxContainer *play_hb = memnew(HBoxContainer); menu_hb->add_child(play_hb); From 0f1704c9e88bdb5a975c348a783d6ff8212d1d7c Mon Sep 17 00:00:00 2001 From: Christoph Schroeder Date: Wed, 6 Mar 2019 11:07:24 +0100 Subject: [PATCH 025/113] Fixes Geometry.segment_intersects_circle working only one way. (cherry picked from commit dcbe55a1facc3cc2c6058dda967fcbe9fdc8c7d0) --- core/math/geometry.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/math/geometry.h b/core/math/geometry.h index 4b478b6b165..7347cb742ad 100644 --- a/core/math/geometry.h +++ b/core/math/geometry.h @@ -702,9 +702,11 @@ public: /* if we can assume that the line segment starts outside the circle (e.g. for continuous time collision detection) then the following can be skipped and we can just return the equivalent of res1 */ sqrtterm = Math::sqrt(sqrtterm); real_t res1 = (-b - sqrtterm) / (2 * a); - //real_t res2 = ( -b + sqrtterm ) / (2 * a); + real_t res2 = (-b + sqrtterm) / (2 * a); - return (res1 >= 0 && res1 <= 1) ? res1 : -1; + if (res1 >= 0 && res1 <= 1) return res1; + if (res2 >= 0 && res2 <= 1) return res2; + return -1; } static inline Vector clip_polygon(const Vector &polygon, const Plane &p_plane) { From 794341e0b5d2d48d67b2a6c34f38066157616d55 Mon Sep 17 00:00:00 2001 From: Aaron Winter Date: Sun, 24 Mar 2019 00:59:24 +0100 Subject: [PATCH 026/113] added negative X and negative Y offset to TileMap clang-format added negative X and negative Y offset to TileMap (cherry picked from commit 63e0fd76753375c0ad9aeb374c807c25d3eaa1a2) --- doc/classes/TileMap.xml | 6 ++++++ editor/plugins/tile_map_editor_plugin.cpp | 16 +++++++++------ scene/2d/tile_map.cpp | 24 ++++++++++++++++++----- scene/2d/tile_map.h | 2 ++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/doc/classes/TileMap.xml b/doc/classes/TileMap.xml index 35240e9e6c7..e1b33ecbe3c 100644 --- a/doc/classes/TileMap.xml +++ b/doc/classes/TileMap.xml @@ -321,6 +321,12 @@ Half offset disabled. + + Half offset on the X coordinate (negative). + + + Half offset on the Y coordinate (negative). + Tile origin at its top-left corner. diff --git a/editor/plugins/tile_map_editor_plugin.cpp b/editor/plugins/tile_map_editor_plugin.cpp index 77e36e34f81..fa0de1077f8 100644 --- a/editor/plugins/tile_map_editor_plugin.cpp +++ b/editor/plugins/tile_map_editor_plugin.cpp @@ -1434,9 +1434,9 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { aabb.expand_to(node->world_to_map(xform_inv.xform(screen_size))); Rect2i si = aabb.grow(1.0); - if (node->get_half_offset() != TileMap::HALF_OFFSET_X) { + if (node->get_half_offset() != TileMap::HALF_OFFSET_X && node->get_half_offset() != TileMap::HALF_OFFSET_NEGATIVE_X) { - int max_lines = 2000; //avoid crash if size too smal + int max_lines = 2000; //avoid crash if size too small for (int i = (si.position.x) - 1; i <= (si.position.x + si.size.x); i++) { @@ -1450,7 +1450,7 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { } } else { - int max_lines = 10000; //avoid crash if size too smal + int max_lines = 10000; //avoid crash if size too small for (int i = (si.position.x) - 1; i <= (si.position.x + si.size.x); i++) { @@ -1458,7 +1458,7 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { Vector2 ofs; if (ABS(j) & 1) { - ofs = cell_xf[0] * 0.5; + ofs = cell_xf[0] * (node->get_half_offset() == TileMap::HALF_OFFSET_X ? 0.5 : -0.5); } Vector2 from = xform.xform(node->map_to_world(Vector2(i, j), true) + ofs); @@ -1477,7 +1477,7 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { int max_lines = 10000; //avoid crash if size too smal - if (node->get_half_offset() != TileMap::HALF_OFFSET_Y) { + if (node->get_half_offset() != TileMap::HALF_OFFSET_Y && node->get_half_offset() != TileMap::HALF_OFFSET_NEGATIVE_Y) { for (int i = (si.position.y) - 1; i <= (si.position.y + si.size.y); i++) { @@ -1498,7 +1498,7 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { Vector2 ofs; if (ABS(j) & 1) { - ofs = cell_xf[1] * 0.5; + ofs = cell_xf[1] * (node->get_half_offset() == TileMap::HALF_OFFSET_Y ? 0.5 : -0.5); } Vector2 from = xform.xform(node->map_to_world(Vector2(j, i), true) + ofs); @@ -1539,8 +1539,12 @@ void TileMapEditor::forward_canvas_draw_over_viewport(Control *p_overlay) { for (int i = 0; i < 4; i++) { if (node->get_half_offset() == TileMap::HALF_OFFSET_X && ABS(over_tile.y) & 1) endpoints[i] += cell_xf[0] * 0.5; + if (node->get_half_offset() == TileMap::HALF_OFFSET_NEGATIVE_X && ABS(over_tile.y) & 1) + endpoints[i] += cell_xf[0] * -0.5; if (node->get_half_offset() == TileMap::HALF_OFFSET_Y && ABS(over_tile.x) & 1) endpoints[i] += cell_xf[1] * 0.5; + if (node->get_half_offset() == TileMap::HALF_OFFSET_NEGATIVE_Y && ABS(over_tile.x) & 1) + endpoints[i] += cell_xf[1] * -0.5; endpoints[i] = xform.xform(endpoints[i]); } Color col; diff --git a/scene/2d/tile_map.cpp b/scene/2d/tile_map.cpp index 46b4f0a5d23..3562011bc29 100644 --- a/scene/2d/tile_map.cpp +++ b/scene/2d/tile_map.cpp @@ -1392,15 +1392,17 @@ Vector2 TileMap::_map_to_world(int p_x, int p_y, bool p_ignore_ofs) const { if (!p_ignore_ofs) { switch (half_offset) { - case HALF_OFFSET_X: { + case HALF_OFFSET_X: + case HALF_OFFSET_NEGATIVE_X: { if (ABS(p_y) & 1) { - ret += get_cell_transform()[0] * 0.5; + ret += get_cell_transform()[0] * (half_offset == HALF_OFFSET_X ? 0.5 : -0.5); } } break; - case HALF_OFFSET_Y: { + case HALF_OFFSET_Y: + case HALF_OFFSET_NEGATIVE_Y: { if (ABS(p_x) & 1) { - ret += get_cell_transform()[1] * 0.5; + ret += get_cell_transform()[1] * (half_offset == HALF_OFFSET_Y ? 0.5 : -0.5); } } break; default: {} @@ -1463,11 +1465,21 @@ Vector2 TileMap::world_to_map(const Vector2 &p_pos) const { ret.x -= 0.5; } } break; + case HALF_OFFSET_NEGATIVE_X: { + if (ret.y > 0 ? int(ret.y) & 1 : (int(ret.y) - 1) & 1) { + ret.x += 0.5; + } + } break; case HALF_OFFSET_Y: { if (ret.x > 0 ? int(ret.x) & 1 : (int(ret.x) - 1) & 1) { ret.y -= 0.5; } } break; + case HALF_OFFSET_NEGATIVE_Y: { + if (ret.x > 0 ? int(ret.x) & 1 : (int(ret.x) - 1) & 1) { + ret.y += 0.5; + } + } break; default: {} } @@ -1678,7 +1690,7 @@ void TileMap::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::VECTOR2, "cell_size", PROPERTY_HINT_RANGE, "1,8192,1"), "set_cell_size", "get_cell_size"); ADD_PROPERTY(PropertyInfo(Variant::INT, "cell_quadrant_size", PROPERTY_HINT_RANGE, "1,128,1"), "set_quadrant_size", "get_quadrant_size"); ADD_PROPERTY(PropertyInfo(Variant::TRANSFORM2D, "cell_custom_transform"), "set_custom_transform", "get_custom_transform"); - ADD_PROPERTY(PropertyInfo(Variant::INT, "cell_half_offset", PROPERTY_HINT_ENUM, "Offset X,Offset Y,Disabled"), "set_half_offset", "get_half_offset"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "cell_half_offset", PROPERTY_HINT_ENUM, "Offset X,Offset Y,Disabled,Offset Negative X,Offset Negative Y"), "set_half_offset", "get_half_offset"); ADD_PROPERTY(PropertyInfo(Variant::INT, "cell_tile_origin", PROPERTY_HINT_ENUM, "Top Left,Center,Bottom Left"), "set_tile_origin", "get_tile_origin"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "cell_y_sort"), "set_y_sort_mode", "is_y_sort_mode_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "cell_clip_uv"), "set_clip_uv", "get_clip_uv"); @@ -1704,6 +1716,8 @@ void TileMap::_bind_methods() { BIND_ENUM_CONSTANT(HALF_OFFSET_X); BIND_ENUM_CONSTANT(HALF_OFFSET_Y); BIND_ENUM_CONSTANT(HALF_OFFSET_DISABLED); + BIND_ENUM_CONSTANT(HALF_OFFSET_NEGATIVE_X); + BIND_ENUM_CONSTANT(HALF_OFFSET_NEGATIVE_Y); BIND_ENUM_CONSTANT(TILE_ORIGIN_TOP_LEFT); BIND_ENUM_CONSTANT(TILE_ORIGIN_CENTER); diff --git a/scene/2d/tile_map.h b/scene/2d/tile_map.h index e450e1e2564..6a1467aa48f 100644 --- a/scene/2d/tile_map.h +++ b/scene/2d/tile_map.h @@ -52,6 +52,8 @@ public: HALF_OFFSET_X, HALF_OFFSET_Y, HALF_OFFSET_DISABLED, + HALF_OFFSET_NEGATIVE_X, + HALF_OFFSET_NEGATIVE_Y, }; enum TileOrigin { From 66c1b8adc454249cb8d36364d74f62e4320a5248 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 26 Mar 2019 20:45:08 +0200 Subject: [PATCH 027/113] Light2D height range expanded to -2048 to 2048. (cherry picked from commit 307bc1652317f8a95181cb4f41a046dc1cfa071c) --- scene/2d/light_2d.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/2d/light_2d.cpp b/scene/2d/light_2d.cpp index d1fce740851..6b12db9e441 100644 --- a/scene/2d/light_2d.cpp +++ b/scene/2d/light_2d.cpp @@ -438,7 +438,7 @@ void Light2D::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::REAL, "energy", PROPERTY_HINT_RANGE, "0.01,100,0.01"), "set_energy", "get_energy"); ADD_PROPERTY(PropertyInfo(Variant::INT, "mode", PROPERTY_HINT_ENUM, "Add,Sub,Mix,Mask"), "set_mode", "get_mode"); ADD_GROUP("Range", "range_"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "range_height", PROPERTY_HINT_RANGE, "-100,100,0.1"), "set_height", "get_height"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "range_height", PROPERTY_HINT_RANGE, "-2048,2048,0.1,or_lesser,or_greater"), "set_height", "get_height"); ADD_PROPERTY(PropertyInfo(Variant::INT, "range_z_min", PROPERTY_HINT_RANGE, itos(VS::CANVAS_ITEM_Z_MIN) + "," + itos(VS::CANVAS_ITEM_Z_MAX) + ",1"), "set_z_range_min", "get_z_range_min"); ADD_PROPERTY(PropertyInfo(Variant::INT, "range_z_max", PROPERTY_HINT_RANGE, itos(VS::CANVAS_ITEM_Z_MIN) + "," + itos(VS::CANVAS_ITEM_Z_MAX) + ",1"), "set_z_range_max", "get_z_range_max"); ADD_PROPERTY(PropertyInfo(Variant::INT, "range_layer_min", PROPERTY_HINT_RANGE, "-512,512,1"), "set_layer_range_min", "get_layer_range_min"); From ad1e8069d3df91c715a31205deeabf82fb5ec0ad Mon Sep 17 00:00:00 2001 From: lupoDharkael Date: Mon, 15 Apr 2019 15:51:52 +0200 Subject: [PATCH 028/113] GDScript: add variable shadowing warning (cherry picked from commit 14078fbb826811ca77e0eef3684f2e5d6f55a6b5) --- modules/gdscript/gdscript.cpp | 5 +++++ modules/gdscript/gdscript.h | 1 + modules/gdscript/gdscript_parser.cpp | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index a8b51f699cb..33e99cb82f1 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -1945,6 +1945,10 @@ String GDScriptWarning::get_message() const { CHECK_SYMBOLS(1); return "The local variable '" + symbols[0] + "' is declared but never used in the block."; } break; + case SHADOWED_VARIABLE: { + CHECK_SYMBOLS(2); + return "The local variable '" + symbols[0] + "' is shadowing an already defined variable at line " + symbols[1] + "."; + } break; case UNUSED_CLASS_VARIABLE: { CHECK_SYMBOLS(1); return "The class variable '" + symbols[0] + "' is declared but never used in the script."; @@ -2048,6 +2052,7 @@ String GDScriptWarning::get_name_from_code(Code p_code) { "UNASSIGNED_VARIABLE", "UNASSIGNED_VARIABLE_OP_ASSIGN", "UNUSED_VARIABLE", + "SHADOWED_VARIABLE", "UNUSED_CLASS_VARIABLE", "UNUSED_ARGUMENT", "UNREACHABLE_CODE", diff --git a/modules/gdscript/gdscript.h b/modules/gdscript/gdscript.h index c67e390e32e..1d75d9e2fed 100644 --- a/modules/gdscript/gdscript.h +++ b/modules/gdscript/gdscript.h @@ -273,6 +273,7 @@ struct GDScriptWarning { UNASSIGNED_VARIABLE, // Variable used but never assigned UNASSIGNED_VARIABLE_OP_ASSIGN, // Variable never assigned but used in an assignment operation (+=, *=, etc) UNUSED_VARIABLE, // Local variable is declared but never used + SHADOWED_VARIABLE, // Variable name shadowed by other variable UNUSED_CLASS_VARIABLE, // Class variable is declared but never used in the file UNUSED_ARGUMENT, // Function argument is never used UNREACHABLE_CODE, // Code after a return statement diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index 2360aa03486..ced0d3fdee1 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -7661,6 +7661,11 @@ void GDScriptParser::_check_function_types(FunctionNode *p_function) { if (p_function->arguments_usage[i] == 0 && !p_function->arguments[i].operator String().begins_with("_")) { _add_warning(GDScriptWarning::UNUSED_ARGUMENT, p_function->line, p_function->name, p_function->arguments[i].operator String()); } + for (int j = 0; j < current_class->variables.size(); j++) { + if (current_class->variables[j].identifier == p_function->arguments[i]) { + _add_warning(GDScriptWarning::SHADOWED_VARIABLE, p_function->line, p_function->arguments[i], itos(current_class->variables[j].line)); + } + } #endif // DEBUG_ENABLED } @@ -7734,6 +7739,17 @@ void GDScriptParser::_check_function_types(FunctionNode *p_function) { p_function->return_type.has_type = false; p_function->return_type.may_yield = true; } + +#ifdef DEBUG_ENABLED + for (Map::Element *E = p_function->body->variables.front(); E; E = E->next()) { + LocalVarNode *lv = E->get(); + for (int i = 0; i < current_class->variables.size(); i++) { + if (current_class->variables[i].identifier == lv->name) { + _add_warning(GDScriptWarning::SHADOWED_VARIABLE, lv->line, lv->name, itos(current_class->variables[i].line)); + } + } + } +#endif // DEBUG_ENABLED } void GDScriptParser::_check_class_blocks_types(ClassNode *p_class) { From 717c492eb6f7f25c91ed4daa0d7e50cc97e2f372 Mon Sep 17 00:00:00 2001 From: mashumafi Date: Thu, 7 Nov 2019 23:01:22 -0500 Subject: [PATCH 029/113] Add setting to exclude addons from script warnings (cherry picked from commit d7137a6b7233c6867c717883a61a1c3607ae84e5) --- modules/gdscript/gdscript.cpp | 1 + modules/gdscript/gdscript_parser.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/gdscript/gdscript.cpp b/modules/gdscript/gdscript.cpp index 33e99cb82f1..769d85d5938 100644 --- a/modules/gdscript/gdscript.cpp +++ b/modules/gdscript/gdscript.cpp @@ -2134,6 +2134,7 @@ GDScriptLanguage::GDScriptLanguage() { #ifdef DEBUG_ENABLED GLOBAL_DEF("debug/gdscript/warnings/enable", true); GLOBAL_DEF("debug/gdscript/warnings/treat_warnings_as_errors", false); + GLOBAL_DEF("debug/gdscript/warnings/exclude_addons", true); GLOBAL_DEF("debug/gdscript/completion/autocomplete_setters_and_getters", false); for (int i = 0; i < (int)GDScriptWarning::WARNING_MAX; i++) { String warning = GDScriptWarning::get_name_from_code((GDScriptWarning::Code)i).to_lower(); diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index ced0d3fdee1..9ba15ec6a4a 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -8179,6 +8179,9 @@ void GDScriptParser::_add_warning(int p_code, int p_line, const String &p_symbol } void GDScriptParser::_add_warning(int p_code, int p_line, const Vector &p_symbols) { + if (GLOBAL_GET("debug/gdscript/warnings/exclude_addons").booleanize() && base_path.begins_with("res://addons/")) { + return; + } if (tokenizer->is_ignoring_warnings() || !GLOBAL_GET("debug/gdscript/warnings/enable").booleanize()) { return; } From 8ef505296747aa64a756213a5dc43f314b66d545 Mon Sep 17 00:00:00 2001 From: Guilherme Felipe Date: Mon, 15 Apr 2019 12:30:20 -0300 Subject: [PATCH 030/113] Add Input::get_current_cursor_shape [Clean up] Removed unused/unnecessary methods. (cherry picked from commit 1bae73d7d060eeb9531fc5ba46de5d31dc7073c7) --- core/os/input.cpp | 1 + core/os/input.h | 4 ++-- core/os/os.cpp | 10 ++++++++++ core/os/os.h | 5 +++-- main/input_default.cpp | 23 +++++++---------------- main/input_default.h | 4 ++-- platform/android/os_android.cpp | 8 -------- platform/android/os_android.h | 3 --- platform/haiku/os_haiku.cpp | 4 ++++ platform/haiku/os_haiku.h | 1 + platform/iphone/os_iphone.cpp | 6 ------ platform/iphone/os_iphone.h | 3 --- platform/osx/os_osx.h | 1 + platform/osx/os_osx.mm | 9 +++++---- platform/server/os_server.cpp | 6 ------ platform/server/os_server.h | 3 --- platform/uwp/os_uwp.cpp | 5 +++++ platform/uwp/os_uwp.h | 1 + platform/windows/os_windows.cpp | 9 +++++---- platform/windows/os_windows.h | 1 + platform/x11/os_x11.cpp | 9 +++++---- platform/x11/os_x11.h | 1 + 22 files changed, 54 insertions(+), 63 deletions(-) diff --git a/core/os/input.cpp b/core/os/input.cpp index 1785356631a..5df1f3a4609 100644 --- a/core/os/input.cpp +++ b/core/os/input.cpp @@ -90,6 +90,7 @@ void Input::_bind_methods() { ClassDB::bind_method(D_METHOD("action_press", "action", "strength"), &Input::action_press, DEFVAL(1.f)); ClassDB::bind_method(D_METHOD("action_release", "action"), &Input::action_release); ClassDB::bind_method(D_METHOD("set_default_cursor_shape", "shape"), &Input::set_default_cursor_shape, DEFVAL(CURSOR_ARROW)); + ClassDB::bind_method(D_METHOD("get_current_cursor_shape"), &Input::get_current_cursor_shape); ClassDB::bind_method(D_METHOD("set_custom_mouse_cursor", "image", "shape", "hotspot"), &Input::set_custom_mouse_cursor, DEFVAL(CURSOR_ARROW), DEFVAL(Vector2())); ClassDB::bind_method(D_METHOD("parse_input_event", "event"), &Input::parse_input_event); ClassDB::bind_method(D_METHOD("set_use_accumulated_input", "enable"), &Input::set_use_accumulated_input); diff --git a/core/os/input.h b/core/os/input.h index e020820361c..a12ded176b6 100644 --- a/core/os/input.h +++ b/core/os/input.h @@ -122,10 +122,10 @@ public: virtual bool is_emulating_touch_from_mouse() const = 0; virtual bool is_emulating_mouse_from_touch() const = 0; - virtual CursorShape get_default_cursor_shape() = 0; + virtual CursorShape get_default_cursor_shape() const = 0; virtual void set_default_cursor_shape(CursorShape p_shape) = 0; + virtual CursorShape get_current_cursor_shape() const = 0; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape = CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()) = 0; - virtual void set_mouse_in_window(bool p_in_window) = 0; virtual String get_joy_button_string(int p_button) = 0; virtual String get_joy_axis_string(int p_axis) = 0; diff --git a/core/os/os.cpp b/core/os/os.cpp index 95b65e6af5f..86941e48036 100644 --- a/core/os/os.cpp +++ b/core/os/os.cpp @@ -225,6 +225,16 @@ int OS::get_virtual_keyboard_height() const { return 0; } +void OS::set_cursor_shape(CursorShape p_shape) { +} + +OS::CursorShape OS::get_cursor_shape() const { + return CURSOR_ARROW; +} + +void OS::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { +} + void OS::print_all_resources(String p_to_file) { ERR_FAIL_COND(p_to_file != "" && _OSPRF); diff --git a/core/os/os.h b/core/os/os.h index 7a987603305..4403bee2b8c 100644 --- a/core/os/os.h +++ b/core/os/os.h @@ -378,8 +378,9 @@ public: // returns height of the currently shown virtual keyboard (0 if keyboard is hidden) virtual int get_virtual_keyboard_height() const; - virtual void set_cursor_shape(CursorShape p_shape) = 0; - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) = 0; + virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; + virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual bool get_swap_ok_cancel() { return false; } virtual void dump_memory_to_file(const char *p_file); diff --git a/main/input_default.cpp b/main/input_default.cpp index 02cc91df128..a5d4bc68918 100644 --- a/main/input_default.cpp +++ b/main/input_default.cpp @@ -627,7 +627,8 @@ bool InputDefault::is_emulating_mouse_from_touch() const { return emulate_mouse_from_touch; } -Input::CursorShape InputDefault::get_default_cursor_shape() { +Input::CursorShape InputDefault::get_default_cursor_shape() const { + return default_shape; } @@ -646,6 +647,11 @@ void InputDefault::set_default_cursor_shape(CursorShape p_shape) { parse_input_event(mm); } +Input::CursorShape InputDefault::get_current_cursor_shape() const { + + return (Input::CursorShape)OS::get_singleton()->get_cursor_shape(); +} + void InputDefault::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (Engine::get_singleton()->is_editor_hint()) return; @@ -653,21 +659,6 @@ void InputDefault::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_sh OS::get_singleton()->set_custom_mouse_cursor(p_cursor, (OS::CursorShape)p_shape, p_hotspot); } -void InputDefault::set_mouse_in_window(bool p_in_window) { - /* no longer supported, leaving this for reference to anyone who might want to implement hardware cursors - if (custom_cursor.is_valid()) { - - if (p_in_window) { - set_mouse_mode(MOUSE_MODE_HIDDEN); - VisualServer::get_singleton()->cursor_set_visible(true); - } else { - set_mouse_mode(MOUSE_MODE_VISIBLE); - VisualServer::get_singleton()->cursor_set_visible(false); - } - } - */ -} - void InputDefault::accumulate_input_event(const Ref &p_event) { ERR_FAIL_COND(p_event.is_null()); diff --git a/main/input_default.h b/main/input_default.h index d82954d1992..91087e4b621 100644 --- a/main/input_default.h +++ b/main/input_default.h @@ -243,10 +243,10 @@ public: void set_emulate_mouse_from_touch(bool p_emulate); virtual bool is_emulating_mouse_from_touch() const; - virtual CursorShape get_default_cursor_shape(); + virtual CursorShape get_default_cursor_shape() const; virtual void set_default_cursor_shape(CursorShape p_shape); + virtual CursorShape get_current_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape = Input::CURSOR_ARROW, const Vector2 &p_hotspot = Vector2()); - virtual void set_mouse_in_window(bool p_in_window); void parse_mapping(String p_mapping); void joy_button(int p_device, int p_button, bool p_pressed); diff --git a/platform/android/os_android.cpp b/platform/android/os_android.cpp index 587bcdd48c5..7ac6cd5b499 100644 --- a/platform/android/os_android.cpp +++ b/platform/android/os_android.cpp @@ -287,14 +287,6 @@ bool OS_Android::can_draw() const { return true; //always? } -void OS_Android::set_cursor_shape(CursorShape p_shape) { - - //android really really really has no mouse.. how amazing.. -} - -void OS_Android::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { -} - void OS_Android::main_loop_begin() { if (main_loop) diff --git a/platform/android/os_android.h b/platform/android/os_android.h index 8df621a6aab..375753bfb59 100644 --- a/platform/android/os_android.h +++ b/platform/android/os_android.h @@ -187,9 +187,6 @@ public: virtual bool can_draw() const; - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - void main_loop_begin(); bool main_loop_iterate(); void main_loop_request_go_back(); diff --git a/platform/haiku/os_haiku.cpp b/platform/haiku/os_haiku.cpp index a6d5a008529..f3fed6669b6 100644 --- a/platform/haiku/os_haiku.cpp +++ b/platform/haiku/os_haiku.cpp @@ -203,6 +203,10 @@ void OS_Haiku::set_cursor_shape(CursorShape p_shape) { //ERR_PRINT("set_cursor_shape() NOT IMPLEMENTED"); } +OS::CursorShape OS_Haiku::get_cursor_shape() const { + // TODO: implement get_cursor_shape +} + void OS_Haiku::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { // TODO } diff --git a/platform/haiku/os_haiku.h b/platform/haiku/os_haiku.h index d7eac106351..6ab006843a0 100644 --- a/platform/haiku/os_haiku.h +++ b/platform/haiku/os_haiku.h @@ -86,6 +86,7 @@ public: virtual Point2 get_mouse_position() const; virtual int get_mouse_button_state() const; virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual int get_screen_count() const; diff --git a/platform/iphone/os_iphone.cpp b/platform/iphone/os_iphone.cpp index 363d67a8383..a2020b74101 100644 --- a/platform/iphone/os_iphone.cpp +++ b/platform/iphone/os_iphone.cpp @@ -491,17 +491,11 @@ void OSIPhone::set_keep_screen_on(bool p_enabled) { _set_keep_screen_on(p_enabled); }; -void OSIPhone::set_cursor_shape(CursorShape p_shape){ - -}; - String OSIPhone::get_user_data_dir() const { return data_dir; }; -void OSIPhone::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot){}; - String OSIPhone::get_name() { return "iOS"; diff --git a/platform/iphone/os_iphone.h b/platform/iphone/os_iphone.h index 9d07bf73130..ce663c34ebd 100644 --- a/platform/iphone/os_iphone.h +++ b/platform/iphone/os_iphone.h @@ -167,9 +167,6 @@ public: virtual void hide_virtual_keyboard(); virtual int get_virtual_keyboard_height() const; - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - virtual Size2 get_window_size() const; virtual Rect2 get_window_safe_area() const; diff --git a/platform/osx/os_osx.h b/platform/osx/os_osx.h index dfe7b27bd04..125a88ab6db 100644 --- a/platform/osx/os_osx.h +++ b/platform/osx/os_osx.h @@ -172,6 +172,7 @@ public: virtual Error open_dynamic_library(const String p_path, void *&p_library_handle, bool p_also_set_library_path = false); virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); virtual void set_mouse_show(bool p_show); diff --git a/platform/osx/os_osx.mm b/platform/osx/os_osx.mm index a03e517f39b..07e549ac8f8 100644 --- a/platform/osx/os_osx.mm +++ b/platform/osx/os_osx.mm @@ -714,8 +714,6 @@ static void _mouseDownEvent(NSEvent *event, int index, int mask, bool pressed) { if (OS_OSX::singleton->main_loop && OS_OSX::singleton->mouse_mode != OS::MOUSE_MODE_CAPTURED) OS_OSX::singleton->main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (OS_OSX::singleton->input) - OS_OSX::singleton->input->set_mouse_in_window(false); } - (void)mouseEntered:(NSEvent *)event { @@ -723,8 +721,6 @@ static void _mouseDownEvent(NSEvent *event, int index, int mask, bool pressed) { return; if (OS_OSX::singleton->main_loop && OS_OSX::singleton->mouse_mode != OS::MOUSE_MODE_CAPTURED) OS_OSX::singleton->main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (OS_OSX::singleton->input) - OS_OSX::singleton->input->set_mouse_in_window(true); OS::CursorShape p_shape = OS_OSX::singleton->cursor_shape; OS_OSX::singleton->cursor_shape = OS::CURSOR_MAX; @@ -1700,6 +1696,11 @@ void OS_OSX::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_OSX::get_cursor_shape() const { + + return cursor_shape; +} + void OS_OSX::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref texture = p_cursor; diff --git a/platform/server/os_server.cpp b/platform/server/os_server.cpp index e643d3e8bb5..53f2a65c8e1 100644 --- a/platform/server/os_server.cpp +++ b/platform/server/os_server.cpp @@ -198,12 +198,6 @@ String OS_Server::get_name() { void OS_Server::move_window_to_foreground() { } -void OS_Server::set_cursor_shape(CursorShape p_shape) { -} - -void OS_Server::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { -} - OS::PowerState OS_Server::get_power_state() { return power_manager->get_power_state(); } diff --git a/platform/server/os_server.h b/platform/server/os_server.h index eebe8ae7771..74410647909 100644 --- a/platform/server/os_server.h +++ b/platform/server/os_server.h @@ -95,9 +95,6 @@ protected: public: virtual String get_name(); - virtual void set_cursor_shape(CursorShape p_shape); - virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); - virtual void set_mouse_show(bool p_show); virtual void set_mouse_grab(bool p_grab); virtual bool is_mouse_grab_enabled() const; diff --git a/platform/uwp/os_uwp.cpp b/platform/uwp/os_uwp.cpp index bc74da8a1b7..82f09032f55 100644 --- a/platform/uwp/os_uwp.cpp +++ b/platform/uwp/os_uwp.cpp @@ -704,6 +704,11 @@ void OS_UWP::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_UWP::get_cursor_shape() const { + + return cursor_shape; +} + void OS_UWP::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { // TODO } diff --git a/platform/uwp/os_uwp.h b/platform/uwp/os_uwp.h index fd78b3cdf77..00f79efb040 100644 --- a/platform/uwp/os_uwp.h +++ b/platform/uwp/os_uwp.h @@ -219,6 +219,7 @@ public: virtual String get_clipboard() const; void set_cursor_shape(CursorShape p_shape); + CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void set_icon(const Ref &p_icon); diff --git a/platform/windows/os_windows.cpp b/platform/windows/os_windows.cpp index 9fe8eca6f56..f988788c58f 100644 --- a/platform/windows/os_windows.cpp +++ b/platform/windows/os_windows.cpp @@ -385,8 +385,6 @@ LRESULT OS_Windows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) outside = true; if (main_loop && mouse_mode != MOUSE_MODE_CAPTURED) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (input) - input->set_mouse_in_window(false); } break; case WM_INPUT: { @@ -481,8 +479,6 @@ LRESULT OS_Windows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) if (main_loop && mouse_mode != MOUSE_MODE_CAPTURED) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (input) - input->set_mouse_in_window(true); CursorShape c = cursor_shape; cursor_shape = CURSOR_MAX; @@ -2298,6 +2294,11 @@ void OS_Windows::set_cursor_shape(CursorShape p_shape) { cursor_shape = p_shape; } +OS::CursorShape OS_Windows::get_cursor_shape() const { + + return cursor_shape; +} + void OS_Windows::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref texture = p_cursor; diff --git a/platform/windows/os_windows.h b/platform/windows/os_windows.h index 2d03532c69f..c15e1cabc31 100644 --- a/platform/windows/os_windows.h +++ b/platform/windows/os_windows.h @@ -273,6 +273,7 @@ public: virtual String get_clipboard() const; void set_cursor_shape(CursorShape p_shape); + CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void GetMaskBitmaps(HBITMAP hSourceBitmap, COLORREF clrTransparent, OUT HBITMAP &hAndMaskBitmap, OUT HBITMAP &hXorMaskBitmap); void set_icon(const Ref &p_icon); diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp index 397e8a25bd4..ebd4b0cc4da 100644 --- a/platform/x11/os_x11.cpp +++ b/platform/x11/os_x11.cpp @@ -2045,15 +2045,11 @@ void OS_X11::process_xevents() { case LeaveNotify: { if (main_loop && !mouse_mode_grab) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_EXIT); - if (input) - input->set_mouse_in_window(false); } break; case EnterNotify: { if (main_loop && !mouse_mode_grab) main_loop->notification(MainLoop::NOTIFICATION_WM_MOUSE_ENTER); - if (input) - input->set_mouse_in_window(true); } break; case FocusIn: minimized = false; @@ -2726,6 +2722,11 @@ void OS_X11::set_cursor_shape(CursorShape p_shape) { current_cursor = p_shape; } +OS::CursorShape OS_X11::get_cursor_shape() const { + + return current_cursor; +} + void OS_X11::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot) { if (p_cursor.is_valid()) { Ref texture = p_cursor; diff --git a/platform/x11/os_x11.h b/platform/x11/os_x11.h index 6d1a66af84e..a54851d4e73 100644 --- a/platform/x11/os_x11.h +++ b/platform/x11/os_x11.h @@ -219,6 +219,7 @@ public: virtual String get_name(); virtual void set_cursor_shape(CursorShape p_shape); + virtual CursorShape get_cursor_shape() const; virtual void set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, const Vector2 &p_hotspot); void set_mouse_mode(MouseMode p_mode); From 7577dd804afc71998367691f6d8029309664c99c Mon Sep 17 00:00:00 2001 From: James Buck Date: Thu, 25 Apr 2019 11:19:04 -0500 Subject: [PATCH 031/113] Cancel tooltip when hiding the node it belongs to (cherry picked from commit bc7863775c994009bddd28c8a96fb1845926371f) --- scene/main/viewport.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scene/main/viewport.cpp b/scene/main/viewport.cpp index 991f4a18957..e31939215e6 100644 --- a/scene/main/viewport.cpp +++ b/scene/main/viewport.cpp @@ -2478,11 +2478,7 @@ void Viewport::_gui_hid_control(Control *p_control) { if (gui.mouse_over == p_control) gui.mouse_over = NULL; if (gui.tooltip == p_control) - gui.tooltip = NULL; - if (gui.tooltip == p_control) { - gui.tooltip = NULL; _gui_cancel_tooltip(); - } } void Viewport::_gui_remove_control(Control *p_control) { From c78c38d817aaecdc033aaba829fd4c7b882bc79d Mon Sep 17 00:00:00 2001 From: Ignacio Etcheverry Date: Fri, 26 Apr 2019 01:19:54 +0200 Subject: [PATCH 032/113] Mono: Lazily load scripts metadata file - Only load the scripts metadata file when it's really needed. This way we avoid false errors, when there is no C# project, about missing scripts metadata file. (cherry picked from commit 791e1294c33626adad323f292702221cc09c21cc) --- modules/mono/csharp_script.cpp | 9 +++++++-- modules/mono/csharp_script.h | 15 ++++++++++++--- modules/mono/editor/csharp_project.cpp | 2 +- modules/mono/mono_gd/gd_mono.cpp | 4 +--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/modules/mono/csharp_script.cpp b/modules/mono/csharp_script.cpp index c8d4f4818ef..7f67c2acfde 100644 --- a/modules/mono/csharp_script.cpp +++ b/modules/mono/csharp_script.cpp @@ -919,7 +919,7 @@ void CSharpLanguage::reload_assemblies(bool p_soft_reload) { } #endif -void CSharpLanguage::project_assembly_loaded() { +void CSharpLanguage::_load_scripts_metadata() { scripts_metadata.clear(); @@ -953,6 +953,7 @@ void CSharpLanguage::project_assembly_loaded() { } scripts_metadata = old_dict_var.operator Dictionary(); + scripts_metadata_invalidated = false; print_verbose("Successfully loaded scripts metadata"); } else { @@ -1024,11 +1025,13 @@ bool CSharpLanguage::debug_break(const String &p_error, bool p_allow_continue) { } } -void CSharpLanguage::_uninitialize_script_bindings() { +void CSharpLanguage::_on_scripts_domain_unloaded() { for (Map::Element *E = script_bindings.front(); E; E = E->next()) { CSharpScriptBinding &script_binding = E->value(); script_binding.inited = false; } + + scripts_metadata_invalidated = true; } void CSharpLanguage::set_language_index(int p_idx) { @@ -1086,6 +1089,8 @@ CSharpLanguage::CSharpLanguage() { #endif lang_idx = -1; + + scripts_metadata_invalidated = true; } CSharpLanguage::~CSharpLanguage() { diff --git a/modules/mono/csharp_script.h b/modules/mono/csharp_script.h index 050527d52b2..fe4eed2e243 100644 --- a/modules/mono/csharp_script.h +++ b/modules/mono/csharp_script.h @@ -309,14 +309,17 @@ class CSharpLanguage : public ScriptLanguage { int lang_idx; Dictionary scripts_metadata; + bool scripts_metadata_invalidated; // For debug_break and debug_break_parse int _debug_parse_err_line; String _debug_parse_err_file; String _debug_error; + void _load_scripts_metadata(); + friend class GDMono; - void _uninitialize_script_bindings(); + void _on_scripts_domain_unloaded(); public: StringNameCache string_names; @@ -341,9 +344,15 @@ public: void reload_assemblies(bool p_soft_reload); #endif - void project_assembly_loaded(); + _FORCE_INLINE_ Dictionary get_scripts_metadata_or_nothing() { + return scripts_metadata_invalidated ? Dictionary() : scripts_metadata; + } - _FORCE_INLINE_ const Dictionary &get_scripts_metadata() { return scripts_metadata; } + _FORCE_INLINE_ const Dictionary &get_scripts_metadata() { + if (scripts_metadata_invalidated) + _load_scripts_metadata(); + return scripts_metadata; + } virtual String get_name() const; diff --git a/modules/mono/editor/csharp_project.cpp b/modules/mono/editor/csharp_project.cpp index beeff51bc25..fe792865563 100644 --- a/modules/mono/editor/csharp_project.cpp +++ b/modules/mono/editor/csharp_project.cpp @@ -158,7 +158,7 @@ Error generate_scripts_metadata(const String &p_project_path, const String &p_ou PoolStringArray project_files = GDMonoMarshal::mono_array_to_PoolStringArray(ret); PoolStringArray::Read r = project_files.read(); - Dictionary old_dict = CSharpLanguage::get_singleton()->get_scripts_metadata(); + Dictionary old_dict = CSharpLanguage::get_singleton()->get_scripts_metadata_or_nothing(); Dictionary new_dict; for (int i = 0; i < project_files.size(); i++) { diff --git a/modules/mono/mono_gd/gd_mono.cpp b/modules/mono/mono_gd/gd_mono.cpp index dfe5d69905e..f70dad9c84b 100644 --- a/modules/mono/mono_gd/gd_mono.cpp +++ b/modules/mono/mono_gd/gd_mono.cpp @@ -665,8 +665,6 @@ bool GDMono::_load_project_assembly() { if (success) { mono_assembly_set_main(project_assembly->get_assembly()); - - CSharpLanguage::get_singleton()->project_assembly_loaded(); } else { if (OS::get_singleton()->is_stdout_verbose()) print_error("Mono: Failed to load project assembly"); @@ -873,7 +871,7 @@ Error GDMono::reload_scripts_domain() { } } - CSharpLanguage::get_singleton()->_uninitialize_script_bindings(); + CSharpLanguage::get_singleton()->_on_scripts_domain_unloaded(); Error err = _load_scripts_domain(); if (err != OK) { From 4f09d433d8c778c3834b5459d98d0838449cdb22 Mon Sep 17 00:00:00 2001 From: Michele Valente Date: Fri, 26 Apr 2019 15:48:10 +0200 Subject: [PATCH 033/113] fix builds made with Clang on some Linux distros This is the same as #23542 (Fix binaries incorrectly detected as shared libraries on some linux distros) but for Clang. It should be fine with Clang 4 or higher. (cherry picked from commit c0fcd55f96a237941ac838fdd38b3ebf2630b107) --- platform/x11/detect.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/platform/x11/detect.py b/platform/x11/detect.py index 2f55faf3809..28f5335e039 100644 --- a/platform/x11/detect.py +++ b/platform/x11/detect.py @@ -1,7 +1,7 @@ import os import platform import sys -from methods import get_compiler_version, using_gcc +from methods import get_compiler_version, using_gcc, using_clang def is_active(): @@ -161,6 +161,12 @@ def configure(env): if version != None and version[0] >= '6': env.Append(CCFLAGS=['-fpie']) env.Append(LINKFLAGS=['-no-pie']) + # Do the same for clang should be fine with Clang 4 and higher + if using_clang(env): + version = get_compiler_version(env) + if version != None and version[0] >= '4': + env.Append(CCFLAGS=['-fpie']) + env.Append(LINKFLAGS=['-no-pie']) ## Dependencies From 918e120da3f4e1f2188eeb3b5b4f5d0cbabc028d Mon Sep 17 00:00:00 2001 From: homer666 Date: Sun, 28 Apr 2019 07:56:04 +1000 Subject: [PATCH 034/113] Update edited keyframe's snap mode properly (cherry picked from commit 62e02629510902ee51dd36a5d4dd7ddeec158831) --- editor/animation_track_editor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/editor/animation_track_editor.cpp b/editor/animation_track_editor.cpp index 87ebef228cf..d9bff1e0e68 100644 --- a/editor/animation_track_editor.cpp +++ b/editor/animation_track_editor.cpp @@ -544,7 +544,7 @@ public: if (use_fps && animation->get_step() > 0) { float max_frame = animation->get_length() / animation->get_step(); - p_list->push_back(PropertyInfo(Variant::REAL, "frame", PROPERTY_HINT_RANGE, "0," + rtos(max_frame) + ",0.01")); + p_list->push_back(PropertyInfo(Variant::REAL, "frame", PROPERTY_HINT_RANGE, "0," + rtos(max_frame) + ",1")); } else { p_list->push_back(PropertyInfo(Variant::REAL, "time", PROPERTY_HINT_RANGE, "0," + rtos(animation->get_length()) + ",0.01")); } @@ -4122,6 +4122,7 @@ void AnimationTrackEditor::_update_key_edit() { key_edit = memnew(AnimationTrackKeyEdit); key_edit->animation = animation; key_edit->track = selection.front()->key().track; + key_edit->use_fps = timeline->is_using_fps(); float ofs = animation->track_get_key_time(key_edit->track, selection.front()->key().key); key_edit->key_ofs = ofs; From 027e2e90fded305c8ac45e3193d31d72382b40e9 Mon Sep 17 00:00:00 2001 From: homer666 Date: Mon, 29 Apr 2019 11:38:45 +1000 Subject: [PATCH 035/113] Refresh import dock on change "animation/storage" (cherry picked from commit 106d1f3c3c07e8d779aa033c0b7cae6a6d02daf3) --- editor/import/resource_importer_scene.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editor/import/resource_importer_scene.cpp b/editor/import/resource_importer_scene.cpp index 9badc366108..26ba98888d7 100644 --- a/editor/import/resource_importer_scene.cpp +++ b/editor/import/resource_importer_scene.cpp @@ -1094,7 +1094,7 @@ void ResourceImporterScene::get_import_options(List *r_options, in r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "animation/import", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_UPDATE_ALL_IF_MODIFIED), true)); r_options->push_back(ImportOption(PropertyInfo(Variant::REAL, "animation/fps", PROPERTY_HINT_RANGE, "1,120,1"), 15)); r_options->push_back(ImportOption(PropertyInfo(Variant::STRING, "animation/filter_script", PROPERTY_HINT_MULTILINE_TEXT), "")); - r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "animation/storage", PROPERTY_HINT_ENUM, "Built-In,Files"), animations_out ? true : false)); + r_options->push_back(ImportOption(PropertyInfo(Variant::INT, "animation/storage", PROPERTY_HINT_ENUM, "Built-In,Files", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_UPDATE_ALL_IF_MODIFIED), animations_out ? true : false)); r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "animation/keep_custom_tracks"), animations_out ? true : false)); r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "animation/optimizer/enabled", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_UPDATE_ALL_IF_MODIFIED), true)); r_options->push_back(ImportOption(PropertyInfo(Variant::REAL, "animation/optimizer/max_linear_error"), 0.05)); From 3afd8a374dca86451b57418d29d8c1a7eda836d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Thu, 2 May 2019 09:22:35 +0200 Subject: [PATCH 036/113] Locales: Add some missing locale names (cherry picked from commit 867b96eb71356d09d530f74b46145260b82f597b) --- core/translation.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/translation.cpp b/core/translation.cpp index ee132476639..a3ff971f455 100644 --- a/core/translation.cpp +++ b/core/translation.cpp @@ -179,6 +179,7 @@ static const char *locale_list[] = { "ff_SN", // Fulah (Senegal) "fi", // Finnish "fi_FI", // Finnish (Finland) + "fil", // Filipino "fil_PH", // Filipino (Philippines) "fo_FO", // Faroese (Faroe Islands) "fr", // French @@ -227,6 +228,7 @@ static const char *locale_list[] = { "ja", // Japanese "ja_JP", // Japanese (Japan) "kab_DZ", // Kabyle (Algeria) + "ka", // Georgian "ka_GE", // Georgian (Georgia) "kk_KZ", // Kazakh (Kazakhstan) "kl_GL", // Kalaallisut (Greenland) @@ -257,10 +259,12 @@ static const char *locale_list[] = { "mg_MG", // Malagasy (Madagascar) "mh_MH", // Marshallese (Marshall Islands) "mhr_RU", // Eastern Mari (Russia) - "mi_NZ", // Maori (New Zealand) + "mi", // Māori + "mi_NZ", // Māori (New Zealand) "miq_NI", // Mískito (Nicaragua) "mk", // Macedonian "mk_MK", // Macedonian (Macedonia) + "ml", // Malayalam "ml_IN", // Malayalam (India) "mni_IN", // Manipuri (India) "mn_MN", // Mongolian (Mongolia) @@ -326,6 +330,7 @@ static const char *locale_list[] = { "sgs_LT", // Samogitian (Lithuania) "shs_CA", // Shuswap (Canada) "sid_ET", // Sidamo (Ethiopia) + "si", // Sinhala "si_LK", // Sinhala (Sri Lanka) "sk", // Slovak "sk_SK", // Slovak (Slovakia) @@ -343,6 +348,7 @@ static const char *locale_list[] = { "sq_MK", // Albanian (Macedonia) "sr", // Serbian "sr_Cyrl", // Serbian (Cyrillic) + "sr_Latn", // Serbian (Latin) "sr_ME", // Serbian (Montenegro) "sr_RS", // Serbian (Serbia) "ss_ZA", // Swati (South Africa) @@ -357,6 +363,7 @@ static const char *locale_list[] = { "ta_IN", // Tamil (India) "ta_LK", // Tamil (Sri Lanka) "tcy_IN", // Tulu (India) + "te", // Telugu "te_IN", // Telugu (India) "tg_TJ", // Tajik (Tajikistan) "the_NP", // Chitwania Tharu (Nepal) @@ -540,6 +547,7 @@ static const char *locale_names[] = { "Fulah (Senegal)", "Finnish", "Finnish (Finland)", + "Filipino", "Filipino (Philippines)", "Faroese (Faroe Islands)", "French", @@ -588,6 +596,7 @@ static const char *locale_names[] = { "Japanese", "Japanese (Japan)", "Kabyle (Algeria)", + "Georgian", "Georgian (Georgia)", "Kazakh (Kazakhstan)", "Kalaallisut (Greenland)", @@ -618,10 +627,12 @@ static const char *locale_names[] = { "Malagasy (Madagascar)", "Marshallese (Marshall Islands)", "Eastern Mari (Russia)", - "Maori (New Zealand)", + "Māori", + "Māori (New Zealand)", "Mískito (Nicaragua)", "Macedonian", "Macedonian (Macedonia)", + "Malayalam", "Malayalam (India)", "Manipuri (India)", "Mongolian (Mongolia)", @@ -687,6 +698,7 @@ static const char *locale_names[] = { "Samogitian (Lithuania)", "Shuswap (Canada)", "Sidamo (Ethiopia)", + "Sinhala", "Sinhala (Sri Lanka)", "Slovak", "Slovak (Slovakia)", @@ -704,6 +716,7 @@ static const char *locale_names[] = { "Albanian (Macedonia)", "Serbian", "Serbian (Cyrillic)", + "Serbian (Latin)", "Serbian (Montenegro)", "Serbian (Serbia)", "Swati (South Africa)", @@ -718,6 +731,7 @@ static const char *locale_names[] = { "Tamil (India)", "Tamil (Sri Lanka)", "Tulu (India)", + "Telugu", "Telugu (India)", "Tajik (Tajikistan)", "Chitwania Tharu (Nepal)", From 2b9c754d1dc62adba3b1bb189ee26e910d988480 Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Thu, 2 May 2019 15:29:07 +0300 Subject: [PATCH 037/113] Fix default values not being read correctly in VS, take two Fixes #25889. (cherry picked from commit b291cfde1b595f236b33f7770ec68042d534051a) --- modules/visual_script/visual_script.cpp | 15 ++------------- modules/visual_script/visual_script.h | 1 - 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/modules/visual_script/visual_script.cpp b/modules/visual_script/visual_script.cpp index 581809fec9f..9f0aa7c9ad9 100644 --- a/modules/visual_script/visual_script.cpp +++ b/modules/visual_script/visual_script.cpp @@ -45,15 +45,7 @@ bool VisualScriptNode::is_breakpoint() const { return breakpoint; } -void VisualScriptNode::_notification(int p_what) { - - if (p_what == NOTIFICATION_POSTINITIALIZE) { - validate_input_default_values(); - } -} - void VisualScriptNode::ports_changed_notify() { - validate_input_default_values(); emit_signal("ports_changed"); } @@ -272,11 +264,7 @@ void VisualScript::_node_ports_changed(int p_id) { Function &func = functions[function]; Ref vsn = func.nodes[p_id].node; - if (OS::get_singleton()->get_main_loop() && - Object::cast_to(OS::get_singleton()->get_main_loop()) && - Engine::get_singleton()->is_editor_hint()) { - vsn->validate_input_default_values(); //force validate default values when editing on editor - } + vsn->validate_input_default_values(); //must revalidate all the functions @@ -352,6 +340,7 @@ void VisualScript::add_node(const StringName &p_func, int p_id, const Ref vsn = p_node; vsn->connect("ports_changed", this, "_node_ports_changed", varray(p_id)); vsn->scripts_used.insert(this); + vsn->validate_input_default_values(); // Validate when fully loaded func.nodes[p_id] = nd; } diff --git a/modules/visual_script/visual_script.h b/modules/visual_script/visual_script.h index 0171b8e6f15..04e2421cfe1 100644 --- a/modules/visual_script/visual_script.h +++ b/modules/visual_script/visual_script.h @@ -54,7 +54,6 @@ class VisualScriptNode : public Resource { void validate_input_default_values(); protected: - void _notification(int p_what); void ports_changed_notify(); static void _bind_methods(); From 4a933a640975894dd929934c92e19a87b1095632 Mon Sep 17 00:00:00 2001 From: Guilherme Felipe Date: Tue, 7 May 2019 12:36:35 -0300 Subject: [PATCH 038/113] [EditorSpatialGizmo] Fix error in intersect_ray (cherry picked from commit 70ead9455ffe65abf505fa4b00cc258049be1580) --- editor/spatial_editor_gizmos.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/editor/spatial_editor_gizmos.cpp b/editor/spatial_editor_gizmos.cpp index 6f379197371..fa9d32f8eb2 100644 --- a/editor/spatial_editor_gizmos.cpp +++ b/editor/spatial_editor_gizmos.cpp @@ -571,9 +571,11 @@ bool EditorSpatialGizmo::intersect_ray(Camera *p_camera, const Point2 &p_point, Transform orig_camera_transform = p_camera->get_camera_transform(); - if (orig_camera_transform.origin.distance_squared_to(t.origin) > 0.01) { + if (orig_camera_transform.origin.distance_squared_to(t.origin) > 0.01 && + ABS(orig_camera_transform.basis.get_axis(Vector3::AXIS_Z).dot(Vector3(0, 1, 0))) < 0.99) { p_camera->look_at(t.origin, Vector3(0, 1, 0)); } + Vector3 c0 = t.xform(Vector3(selectable_icon_size, selectable_icon_size, 0) * scale); Vector3 c1 = t.xform(Vector3(-selectable_icon_size, -selectable_icon_size, 0) * scale); @@ -582,7 +584,7 @@ bool EditorSpatialGizmo::intersect_ray(Camera *p_camera, const Point2 &p_point, p_camera->set_global_transform(orig_camera_transform); - Rect2 rect(p0, p1 - p0); + Rect2 rect(p0, (p1 - p0).abs()); rect.set_position(center - rect.get_size() / 2.0); From 5c9694bc0d7a0e0104871bdd3325fa6fbafb0228 Mon Sep 17 00:00:00 2001 From: Fabio Alessandrelli Date: Fri, 5 Apr 2019 17:13:55 +0200 Subject: [PATCH 039/113] Avoid _can_call_mode resetting error message in MultiplayerAPI (cherry picked from commit bba77fe3879a6233c24289f0201be830edfc8c73) --- core/io/multiplayer_api.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/io/multiplayer_api.cpp b/core/io/multiplayer_api.cpp index 86382939f18..78b7adcb69f 100644 --- a/core/io/multiplayer_api.cpp +++ b/core/io/multiplayer_api.cpp @@ -283,8 +283,9 @@ void MultiplayerAPI::_process_rpc(Node *p_node, const StringName &p_name, int p_ rpc_mode = p_node->get_script_instance()->get_rpc_mode(p_name); } - ERR_EXPLAIN("RPC '" + String(p_name) + "' is not allowed from: " + itos(p_from) + ". Mode is " + itos((int)rpc_mode) + ", master is " + itos(p_node->get_network_master()) + "."); - ERR_FAIL_COND(!_can_call_mode(p_node, rpc_mode, p_from)); + bool can_call = _can_call_mode(p_node, rpc_mode, p_from); + ERR_EXPLAIN("RPC '" + String(p_name) + "' is not allowed on node " + p_node->get_path() + " from: " + itos(p_from) + ". Mode is " + itos((int)rpc_mode) + ", master is " + itos(p_node->get_network_master()) + "."); + ERR_FAIL_COND(!can_call); int argc = p_packet[p_offset]; Vector args; @@ -332,8 +333,9 @@ void MultiplayerAPI::_process_rset(Node *p_node, const StringName &p_name, int p rset_mode = p_node->get_script_instance()->get_rset_mode(p_name); } - ERR_EXPLAIN("RSET '" + String(p_name) + "' is not allowed from: " + itos(p_from) + ". Mode is " + itos((int)rset_mode) + ", master is " + itos(p_node->get_network_master()) + "."); - ERR_FAIL_COND(!_can_call_mode(p_node, rset_mode, p_from)); + bool can_call = _can_call_mode(p_node, rset_mode, p_from); + ERR_EXPLAIN("RSET '" + String(p_name) + "' is not allowed on node " + p_node->get_path() + " from: " + itos(p_from) + ". Mode is " + itos((int)rset_mode) + ", master is " + itos(p_node->get_network_master()) + "."); + ERR_FAIL_COND(!can_call); Variant value; Error err = decode_variant(value, &p_packet[p_offset], p_packet_len - p_offset, NULL, allow_object_decoding || network_peer->is_object_decoding_allowed()); From dc90131d8c0c254624fe38b97720d7f0b13fefad Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Thu, 9 May 2019 18:03:08 +0300 Subject: [PATCH 040/113] Fix AudioEffectRecord messing up the effect stack by not writing to dst_frames (cherry picked from commit 9285aad8b359defaacbbcbf3796a530240c3876b) --- servers/audio/effects/audio_effect_record.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/servers/audio/effects/audio_effect_record.cpp b/servers/audio/effects/audio_effect_record.cpp index 96d5c9df892..abf9d5593ca 100644 --- a/servers/audio/effects/audio_effect_record.cpp +++ b/servers/audio/effects/audio_effect_record.cpp @@ -32,6 +32,9 @@ void AudioEffectRecordInstance::process(const AudioFrame *p_src_frames, AudioFrame *p_dst_frames, int p_frame_count) { if (!is_recording) { + for (int i = 0; i < p_frame_count; i++) { + p_dst_frames[i] = p_src_frames[i]; + } return; } @@ -39,6 +42,7 @@ void AudioEffectRecordInstance::process(const AudioFrame *p_src_frames, AudioFra const AudioFrame *src = p_src_frames; AudioFrame *rb_buf = ring_buffer.ptrw(); for (int i = 0; i < p_frame_count; i++) { + p_dst_frames[i] = p_src_frames[i]; rb_buf[ring_buffer_pos & ring_buffer_mask] = src[i]; ring_buffer_pos++; } From 49613428f1371b52717dc0ff4c1be57080c8b32c Mon Sep 17 00:00:00 2001 From: JFonS Date: Fri, 10 May 2019 10:35:15 +0200 Subject: [PATCH 041/113] Fixes to ClippedCamera This work has been kindly sponsored by IMVU. (cherry picked from commit 9e97bac0cf4baeb6576874bc5ab877f57433a86d) --- scene/3d/camera.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scene/3d/camera.cpp b/scene/3d/camera.cpp index 368cebeeab0..f2a11f3e111 100644 --- a/scene/3d/camera.cpp +++ b/scene/3d/camera.cpp @@ -684,8 +684,9 @@ void ClippedCamera::set_process_mode(ProcessMode p_mode) { if (process_mode == p_mode) { return; } - set_process_internal(p_mode == CLIP_PROCESS_IDLE); - set_physics_process_internal(p_mode == CLIP_PROCESS_PHYSICS); + process_mode = p_mode; + set_process_internal(process_mode == CLIP_PROCESS_IDLE); + set_physics_process_internal(process_mode == CLIP_PROCESS_PHYSICS); } ClippedCamera::ProcessMode ClippedCamera::get_process_mode() const { return process_mode; @@ -748,7 +749,7 @@ void ClippedCamera::_notification(int p_what) { float csafe, cunsafe; if (dspace->cast_motion(pyramid_shape, xf, cam_pos - ray_from, margin, csafe, cunsafe, exclude, collision_mask, clip_to_bodies, clip_to_areas)) { - clip_offset = cam_pos.distance_to(ray_from + (cam_pos - ray_from).normalized() * csafe); + clip_offset = cam_pos.distance_to(ray_from + (cam_pos - ray_from) * csafe); } _update_camera(); From d79be7ec338312259157191f3c0d51e1b9c7387d Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Fri, 10 May 2019 18:03:18 +0300 Subject: [PATCH 042/113] Allow or_greater for most properties of NavMesh Closes #28624 (cherry picked from commit 68bee3d9a262c664834c409c0c97c4cff48e9593) --- scene/3d/navigation_mesh.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scene/3d/navigation_mesh.cpp b/scene/3d/navigation_mesh.cpp index 93731c40231..003f76664de 100644 --- a/scene/3d/navigation_mesh.cpp +++ b/scene/3d/navigation_mesh.cpp @@ -410,19 +410,19 @@ void NavigationMesh::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::INT, "sample_partition_type/sample_partition_type", PROPERTY_HINT_ENUM, "Watershed,Monotone,Layers"), "set_sample_partition_type", "get_sample_partition_type"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "cell/size", PROPERTY_HINT_RANGE, "0.1,1.0,0.01"), "set_cell_size", "get_cell_size"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "cell/height", PROPERTY_HINT_RANGE, "0.1,1.0,0.01"), "set_cell_height", "get_cell_height"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/height", PROPERTY_HINT_RANGE, "0.1,5.0,0.01"), "set_agent_height", "get_agent_height"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/radius", PROPERTY_HINT_RANGE, "0.1,5.0,0.01"), "set_agent_radius", "get_agent_radius"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/max_climb", PROPERTY_HINT_RANGE, "0.1,5.0,0.01"), "set_agent_max_climb", "get_agent_max_climb"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "cell/size", PROPERTY_HINT_RANGE, "0.1,1.0,0.01,or_greater"), "set_cell_size", "get_cell_size"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "cell/height", PROPERTY_HINT_RANGE, "0.1,1.0,0.01,or_greater"), "set_cell_height", "get_cell_height"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/height", PROPERTY_HINT_RANGE, "0.1,5.0,0.01,or_greater"), "set_agent_height", "get_agent_height"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/radius", PROPERTY_HINT_RANGE, "0.1,5.0,0.01,or_greater"), "set_agent_radius", "get_agent_radius"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/max_climb", PROPERTY_HINT_RANGE, "0.1,5.0,0.01,or_greater"), "set_agent_max_climb", "get_agent_max_climb"); ADD_PROPERTY(PropertyInfo(Variant::REAL, "agent/max_slope", PROPERTY_HINT_RANGE, "0.0,90.0,0.1"), "set_agent_max_slope", "get_agent_max_slope"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "region/min_size", PROPERTY_HINT_RANGE, "0.0,150.0,0.01"), "set_region_min_size", "get_region_min_size"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "region/merge_size", PROPERTY_HINT_RANGE, "0.0,150.0,0.01"), "set_region_merge_size", "get_region_merge_size"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "edge/max_length", PROPERTY_HINT_RANGE, "0.0,50.0,0.01"), "set_edge_max_length", "get_edge_max_length"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "edge/max_error", PROPERTY_HINT_RANGE, "0.1,3.0,0.01"), "set_edge_max_error", "get_edge_max_error"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "polygon/verts_per_poly", PROPERTY_HINT_RANGE, "3.0,12.0,1.0"), "set_verts_per_poly", "get_verts_per_poly"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "detail/sample_distance", PROPERTY_HINT_RANGE, "0.0,16.0,0.01"), "set_detail_sample_distance", "get_detail_sample_distance"); - ADD_PROPERTY(PropertyInfo(Variant::REAL, "detail/sample_max_error", PROPERTY_HINT_RANGE, "0.0,16.0,0.01"), "set_detail_sample_max_error", "get_detail_sample_max_error"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "region/min_size", PROPERTY_HINT_RANGE, "0.0,150.0,0.01,or_greater"), "set_region_min_size", "get_region_min_size"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "region/merge_size", PROPERTY_HINT_RANGE, "0.0,150.0,0.01,or_greater"), "set_region_merge_size", "get_region_merge_size"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "edge/max_length", PROPERTY_HINT_RANGE, "0.0,50.0,0.01,or_greater"), "set_edge_max_length", "get_edge_max_length"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "edge/max_error", PROPERTY_HINT_RANGE, "0.1,3.0,0.01,or_greater"), "set_edge_max_error", "get_edge_max_error"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "polygon/verts_per_poly", PROPERTY_HINT_RANGE, "3.0,12.0,1.0,or_greater"), "set_verts_per_poly", "get_verts_per_poly"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "detail/sample_distance", PROPERTY_HINT_RANGE, "0.0,16.0,0.01,or_greater"), "set_detail_sample_distance", "get_detail_sample_distance"); + ADD_PROPERTY(PropertyInfo(Variant::REAL, "detail/sample_max_error", PROPERTY_HINT_RANGE, "0.0,16.0,0.01,or_greater"), "set_detail_sample_max_error", "get_detail_sample_max_error"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "filter/low_hanging_obstacles"), "set_filter_low_hanging_obstacles", "get_filter_low_hanging_obstacles"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "filter/ledge_spans"), "set_filter_ledge_spans", "get_filter_ledge_spans"); From 53eb140cf1f0ba9033f7e6ef026ae4a200e4198c Mon Sep 17 00:00:00 2001 From: Fabio Alessandrelli Date: Sun, 12 May 2019 15:36:38 +0200 Subject: [PATCH 043/113] Fix GDNative library resource loading. Store general properties in ConfigFile too when modifying them. Additionally set config_file property as internal as it's not editable from inspector. It also does not appear to get saved in a meaningful way (saved as Object(ConfigFile, ...)) (cherry picked from commit aecbb330f37a1c80ba3878c50bfd948317845200) --- modules/gdnative/gdnative.cpp | 2 +- modules/gdnative/gdnative.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/gdnative/gdnative.cpp b/modules/gdnative/gdnative.cpp index e8278825bc8..c8e17e8dc53 100644 --- a/modules/gdnative/gdnative.cpp +++ b/modules/gdnative/gdnative.cpp @@ -243,7 +243,7 @@ void GDNativeLibrary::_bind_methods() { ClassDB::bind_method(D_METHOD("set_symbol_prefix", "symbol_prefix"), &GDNativeLibrary::set_symbol_prefix); ClassDB::bind_method(D_METHOD("set_reloadable", "reloadable"), &GDNativeLibrary::set_reloadable); - ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "config_file", PROPERTY_HINT_RESOURCE_TYPE, "ConfigFile"), "set_config_file", "get_config_file"); + ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "config_file", PROPERTY_HINT_RESOURCE_TYPE, "ConfigFile", 0), "set_config_file", "get_config_file"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "load_once"), "set_load_once", "should_load_once"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "singleton"), "set_singleton", "is_singleton"); diff --git a/modules/gdnative/gdnative.h b/modules/gdnative/gdnative.h index 492dc5beaae..ef57387059b 100644 --- a/modules/gdnative/gdnative.h +++ b/modules/gdnative/gdnative.h @@ -99,16 +99,20 @@ public: } _FORCE_INLINE_ void set_load_once(bool p_load_once) { + config_file->set_value("general", "load_once", p_load_once); load_once = p_load_once; } _FORCE_INLINE_ void set_singleton(bool p_singleton) { + config_file->set_value("general", "singleton", p_singleton); singleton = p_singleton; } _FORCE_INLINE_ void set_symbol_prefix(String p_symbol_prefix) { + config_file->set_value("general", "symbol_prefix", p_symbol_prefix); symbol_prefix = p_symbol_prefix; } _FORCE_INLINE_ void set_reloadable(bool p_reloadable) { + config_file->set_value("general", "reloadable", p_reloadable); reloadable = p_reloadable; } From dcc9c1cc2aa3486e963e743121257f4494953dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gilles=20Roudi=C3=A8re?= Date: Wed, 15 May 2019 11:21:10 +0200 Subject: [PATCH 044/113] Fix NaN with get_action_strength (cherry picked from commit 3bfa080c9c6ac13b20670b04d58692aad08fb99b) --- core/os/input_event.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/os/input_event.cpp b/core/os/input_event.cpp index 25a5c2afebf..a0720173534 100644 --- a/core/os/input_event.cpp +++ b/core/os/input_event.cpp @@ -717,8 +717,17 @@ bool InputEventJoypadMotion::action_match(const Ref &p_event, bool * bool pressed = same_direction ? Math::abs(jm->get_axis_value()) >= p_deadzone : false; if (p_pressed != NULL) *p_pressed = pressed; - if (p_strength != NULL) - *p_strength = pressed ? CLAMP(Math::inverse_lerp(p_deadzone, 1.0f, Math::abs(jm->get_axis_value())), 0.0f, 1.0f) : 0.0f; + if (p_strength != NULL) { + if (pressed) { + if (p_deadzone == 1.0f) { + *p_strength = 1.0f; + } else { + *p_strength = CLAMP(Math::inverse_lerp(p_deadzone, 1.0f, Math::abs(jm->get_axis_value())), 0.0f, 1.0f); + } + } else { + *p_strength = 0.0f; + } + } } return match; } From 8ee5dc5850543dba488266ab18e19f6458c31678 Mon Sep 17 00:00:00 2001 From: Frank Secilia Date: Wed, 15 May 2019 15:28:55 -0400 Subject: [PATCH 045/113] Fix indexing failure in NativeScriptLanguage::unregister_binding_functions. binding_functions.size() and an instance's binding_data.size() can get out of sync. They sync up when an instance's bindings are requested. When binding functions are registered after creating an instance's bindings, the instance's bindings are out of sync until requested again. If they're never requested, they're never synced. unregister_binding_functions indexes into binding_data, but only checks that its safe to index into binding_functions. When they're out of sync, indexing fails. This revision checks that it's safe to index into binding_data. (cherry picked from commit a4a0e642456f80b71a6383f7fccd5ea93c07c2ea) --- modules/gdnative/nativescript/nativescript.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gdnative/nativescript/nativescript.cpp b/modules/gdnative/nativescript/nativescript.cpp index 5cf144d4fe3..04ba28dc687 100644 --- a/modules/gdnative/nativescript/nativescript.cpp +++ b/modules/gdnative/nativescript/nativescript.cpp @@ -1309,7 +1309,7 @@ void NativeScriptLanguage::unregister_binding_functions(int p_idx) { for (Set *>::Element *E = binding_instances.front(); E; E = E->next()) { Vector &binding_data = *E->get(); - if (binding_data[p_idx] && binding_functions[p_idx].second.free_instance_binding_data) + if (p_idx < binding_data.size() && binding_data[p_idx] && binding_functions[p_idx].second.free_instance_binding_data) binding_functions[p_idx].second.free_instance_binding_data(binding_functions[p_idx].second.data, binding_data[p_idx]); } From 390dbbbcbde4a5e85282ca713a416c1c547ecd2f Mon Sep 17 00:00:00 2001 From: Fabio Alessandrelli Date: Thu, 16 May 2019 23:22:52 +0200 Subject: [PATCH 046/113] Fix Object::get_indexed for simple properties. Object::get_indexed was not correctly reporting invalid keys if the name was a direct property (not a subproperty), causing for example Tween to not report correctly a bad interpolate_property key. (cherry picked from commit d39ffc101bd9592341530e5bc5436ddab1cd8f99) --- core/object.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/core/object.cpp b/core/object.cpp index 79f0fa0229f..6712409f0ef 100644 --- a/core/object.cpp +++ b/core/object.cpp @@ -608,18 +608,16 @@ Variant Object::get_indexed(const Vector &p_names, bool *r_valid) co } bool valid = false; - Variant current_value = get(p_names[0]); + Variant current_value = get(p_names[0], &valid); for (int i = 1; i < p_names.size(); i++) { current_value = current_value.get_named(p_names[i], &valid); - if (!valid) { - if (r_valid) - *r_valid = false; - return Variant(); - } + if (!valid) + break; } if (r_valid) - *r_valid = true; + *r_valid = valid; + return current_value; } From 909a1cced8bb7ae63c9e9ee6862572b118c7efd1 Mon Sep 17 00:00:00 2001 From: Paulb23 Date: Thu, 16 May 2019 22:32:06 +0100 Subject: [PATCH 047/113] Fix text_edit drawing incorrect chars when syntax highlighting enabled (cherry picked from commit a8a0392c12d8a27f8c0f376f67abf7d3a06987df) --- scene/gui/text_edit.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scene/gui/text_edit.cpp b/scene/gui/text_edit.cpp index fbd60ce2200..631ddaf7de5 100644 --- a/scene/gui/text_edit.cpp +++ b/scene/gui/text_edit.cpp @@ -1009,10 +1009,7 @@ void TextEdit::_notification(int p_what) { } if ((char_ofs + char_margin + char_w) >= xmargin_end) { - if (syntax_coloring) - continue; - else - break; + break; } bool in_search_result = false; From 2da174c9b47a8450600b859ba564c4b5c1015dee Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 17 May 2019 20:49:24 +0200 Subject: [PATCH 048/113] Resolve: #24682 CSGPolygon - weird end cap (cherry picked from commit 89a9f4621f9848f651b73ab6c7c6321075d0c55c) --- modules/csg/csg_shape.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/csg/csg_shape.cpp b/modules/csg/csg_shape.cpp index 775ec67ba64..315ebb44e1d 100644 --- a/modules/csg/csg_shape.cpp +++ b/modules/csg/csg_shape.cpp @@ -2060,6 +2060,9 @@ CSGBrush *CSGPolygon::_build_brush() { for (int i = 0; i <= splits; i++) { float ofs = i * path_interval; + if (ofs > bl) { + ofs = bl; + } if (i == splits && path_joined) { ofs = 0.0; } From ffab947e1c3a51dc0a70692487345d0cfe31a3ce Mon Sep 17 00:00:00 2001 From: RameshRavone Date: Sat, 18 May 2019 07:54:16 +0530 Subject: [PATCH 049/113] NULL global_tag for non exposed classes (cherry picked from commit 71808b38bc787cb2c903f3ac0479a0bc6e905fa8) --- modules/gdnative/nativescript/nativescript.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/gdnative/nativescript/nativescript.cpp b/modules/gdnative/nativescript/nativescript.cpp index 04ba28dc687..c2aa8427b43 100644 --- a/modules/gdnative/nativescript/nativescript.cpp +++ b/modules/gdnative/nativescript/nativescript.cpp @@ -1345,7 +1345,7 @@ void *NativeScriptLanguage::get_instance_binding_data(int p_idx, Object *p_objec if (!(*binding_data)[p_idx]) { - const void *global_type_tag = global_type_tags[p_idx].get(p_object->get_class_name()); + const void *global_type_tag = get_global_type_tag(p_idx, p_object->get_class_name()); // no binding data yet, soooooo alloc new one \o/ (*binding_data).write[p_idx] = binding_functions[p_idx].second.alloc_instance_binding_data(binding_functions[p_idx].second.data, global_type_tag, (godot_object *)p_object); @@ -1454,6 +1454,9 @@ const void *NativeScriptLanguage::get_global_type_tag(int p_idx, StringName p_cl const HashMap &tags = global_type_tags[p_idx]; + if (!tags.has(p_class_name)) + return NULL; + const void *tag = tags.get(p_class_name); return tag; From 6d57ebed1a6b7deda0a258e5b5b44f84d0734ca2 Mon Sep 17 00:00:00 2001 From: Ibrahn Sahir Date: Sun, 19 May 2019 14:05:15 +0100 Subject: [PATCH 050/113] MessageQueue::flush now always destroys parameters of a spent message Previously, destructors of Variant parameters were not called if the target of the message was not found. (cherry picked from commit 5626a1ec20b8df3faa0afd54787d08ccebed927d) --- core/message_queue.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/message_queue.cpp b/core/message_queue.cpp index c57bd4081c5..229cd1261c9 100644 --- a/core/message_queue.cpp +++ b/core/message_queue.cpp @@ -302,10 +302,6 @@ void MessageQueue::flush() { _call_function(target, message->target, args, message->args, message->type & FLAG_SHOW_ERROR); - for (int i = 0; i < message->args; i++) { - args[i].~Variant(); - } - } break; case TYPE_NOTIFICATION: { @@ -319,11 +315,17 @@ void MessageQueue::flush() { // messages don't expect a return value target->set(message->target, *arg); - arg->~Variant(); } break; } } + if ((message->type & FLAG_MASK) != TYPE_NOTIFICATION) { + Variant *args = (Variant *)(message + 1); + for (int i = 0; i < message->args; i++) { + args[i].~Variant(); + } + } + message->~Message(); _THREAD_SAFE_LOCK_ From 7022e02d4dab264c7527c5dbd3bc3d43a3319af6 Mon Sep 17 00:00:00 2001 From: Lukas Zanner Date: Sun, 19 May 2019 19:09:16 +0200 Subject: [PATCH 051/113] Initialize padding on PoolByteArray serialization (cherry picked from commit c35f4e880868131d442394f1c00b572a55d0ee28) --- core/io/marshalls.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/io/marshalls.cpp b/core/io/marshalls.cpp index 19adcc7c430..f70686cf8ee 100644 --- a/core/io/marshalls.cpp +++ b/core/io/marshalls.cpp @@ -1230,11 +1230,15 @@ Error encode_variant(const Variant &p_variant, uint8_t *r_buffer, int &r_len, bo buf += 4; PoolVector::Read r = data.read(); copymem(buf, &r[0], datalen * datasize); + buf += datalen * datasize; } r_len += 4 + datalen * datasize; - while (r_len % 4) + while (r_len % 4) { r_len++; + if (buf) + *(buf++) = 0; + } } break; case Variant::POOL_INT_ARRAY: { From 14587802f9278729b6b19ac2cc12fc1ede7b4ecb Mon Sep 17 00:00:00 2001 From: Guilherme Felipe Date: Sun, 19 May 2019 23:43:26 -0300 Subject: [PATCH 052/113] [StateMachine] Fix error message for travel method (cherry picked from commit 4de6c94eb048a355a12b9ec4cdfaf9d9265f2de2) --- scene/animation/animation_node_state_machine.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scene/animation/animation_node_state_machine.cpp b/scene/animation/animation_node_state_machine.cpp index 22c26f99a33..1c9ea6d8539 100644 --- a/scene/animation/animation_node_state_machine.cpp +++ b/scene/animation/animation_node_state_machine.cpp @@ -315,8 +315,9 @@ float AnimationNodeStateMachinePlayback::process(AnimationNodeStateMachine *sm, if (start_request_travel) { if (!playing) { + String node_name = start_request; start_request = StringName(); - ERR_EXPLAIN("Can't travel to '" + String(start_request) + "' if state machine is not active."); + ERR_EXPLAIN("Can't travel to '" + node_name + "' if state machine is not playing."); ERR_FAIL_V(0); } From 665c00eb58a8b7b51373f9adffcb58918b6105a6 Mon Sep 17 00:00:00 2001 From: Hugo Locurcio Date: Mon, 20 May 2019 14:43:42 +0200 Subject: [PATCH 053/113] Allow empty replacement texts in the Find in Files replace dialog This closes #29026. (cherry picked from commit a411067829dfcdf7081399bfb320218d0bd0bc32) --- editor/find_in_files.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/editor/find_in_files.cpp b/editor/find_in_files.cpp index 4dbba952bf5..11af9fc2eb8 100644 --- a/editor/find_in_files.cpp +++ b/editor/find_in_files.cpp @@ -751,7 +751,6 @@ void FindInFilesPanel::_on_replace_text_changed(String text) { void FindInFilesPanel::_on_replace_all_clicked() { String replace_text = get_replace_text(); - ERR_FAIL_COND(replace_text.empty()); PoolStringArray modified_files; @@ -887,7 +886,7 @@ String FindInFilesPanel::get_replace_text() { void FindInFilesPanel::update_replace_buttons() { String text = get_replace_text(); - bool disabled = text.empty() || _finder->is_searching(); + bool disabled = _finder->is_searching(); _replace_all_button->set_disabled(disabled); } From 949e1da462f7bad5669c5f4d44b7a6339428c72f Mon Sep 17 00:00:00 2001 From: Swarnim Arun Date: Mon, 20 May 2019 22:15:43 +0530 Subject: [PATCH 054/113] Fix history state setup in VisualScript (cherry picked from commit 7155477caa581433b5d5daea42d23f1d924869b5) --- modules/visual_script/visual_script_editor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/visual_script/visual_script_editor.cpp b/modules/visual_script/visual_script_editor.cpp index 7e54891d977..47c073035f1 100644 --- a/modules/visual_script/visual_script_editor.cpp +++ b/modules/visual_script/visual_script_editor.cpp @@ -2040,7 +2040,7 @@ void VisualScriptEditor::set_edit_state(const Variant &p_state) { Dictionary d = p_state; if (d.has("function")) { - edited_func = p_state; + edited_func = d["function"]; selected = edited_func; } From bfcb9dfa01865c7496a214e6c97eddecf4054267 Mon Sep 17 00:00:00 2001 From: Martin Dahlgren Date: Mon, 20 May 2019 23:51:01 +0200 Subject: [PATCH 055/113] Fix getting max magnitude in AudioEffectSpectrumAnalyzer (cherry picked from commit 47f628a013e501b9eefb7ed8d5b23862bc5083ea) --- servers/audio/effects/audio_effect_spectrum_analyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servers/audio/effects/audio_effect_spectrum_analyzer.cpp b/servers/audio/effects/audio_effect_spectrum_analyzer.cpp index 01a52aa01f3..5555dde6af9 100644 --- a/servers/audio/effects/audio_effect_spectrum_analyzer.cpp +++ b/servers/audio/effects/audio_effect_spectrum_analyzer.cpp @@ -170,7 +170,7 @@ Vector2 AudioEffectSpectrumAnalyzerInstance::get_magnitude_for_frequency_range(f for (int i = begin_pos; i <= end_pos; i++) { max.x = MAX(max.x, r[i].l); - max.y = MAX(max.x, r[i].r); + max.y = MAX(max.y, r[i].r); } return max; From 8dce0ce0226246b1fe6f23f89b2fe7ac2d53086b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 21 May 2019 14:32:01 +0200 Subject: [PATCH 056/113] i18n: Skip unsupported locales for editor translations Godot currently can't properly render scripts that require Right-To-Left and font shaping support, so we skip those. This is a temporary measure until these features are supported. Fixes #28577. (cherry picked from commit 92f67ceef329923a0d0e7c34c8ca389dd2b3c6bb) --- editor/editor_settings.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/editor/editor_settings.cpp b/editor/editor_settings.cpp index b9ed63c1b68..f477134f5a1 100644 --- a/editor/editor_settings.cpp +++ b/editor/editor_settings.cpp @@ -268,6 +268,11 @@ void EditorSettings::_load_defaults(Ref p_extra_config) { String host_lang = OS::get_singleton()->get_locale(); host_lang = TranslationServer::standardize_locale(host_lang); + // Some locales are not properly supported currently in Godot due to lack of font shaping + // (e.g. Arabic or Hindi), so even though we have work in progress translations for them, + // we skip them as they don't render properly. (GH-28577) + const Vector locales_to_skip = String("ar,bn,fa,he,hi,ml,si,ta,te,ur").split(","); + String best; EditorTranslationList *etl = _editor_translations; @@ -275,6 +280,15 @@ void EditorSettings::_load_defaults(Ref p_extra_config) { while (etl->data) { const String &locale = etl->lang; + + // Skip locales which we can't render properly (see above comment). + // Test against language code without regional variants (e.g. ur_PK). + String lang_code = locale.get_slice("_", 0); + if (locales_to_skip.find(lang_code) != -1) { + etl++; + continue; + } + lang_hint += ","; lang_hint += locale; From b01ef197a6d3515991ccbdfdfcad8b9a8a7233f0 Mon Sep 17 00:00:00 2001 From: Michael Alexsander Silva Dias Date: Tue, 21 May 2019 13:14:45 -0300 Subject: [PATCH 057/113] Fix bottom panel visibility behaviour in the theme editor Fixes #29034, fixes #29056 (cherry picked from commit b5c8205312c04205db5fe7ebf87ddbaa9dbfc950) --- editor/plugins/theme_editor_plugin.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/editor/plugins/theme_editor_plugin.cpp b/editor/plugins/theme_editor_plugin.cpp index 7eee4b2f47c..5796dcc6c7c 100644 --- a/editor/plugins/theme_editor_plugin.cpp +++ b/editor/plugins/theme_editor_plugin.cpp @@ -878,11 +878,9 @@ ThemeEditor::ThemeEditor() { void ThemeEditorPlugin::edit(Object *p_node) { if (Object::cast_to(p_node)) { - theme_editor->show(); theme_editor->edit(Object::cast_to(p_node)); } else { theme_editor->edit(Ref()); - theme_editor->hide(); } } @@ -897,11 +895,11 @@ void ThemeEditorPlugin::make_visible(bool p_visible) { theme_editor->set_process(true); button->show(); editor->make_bottom_panel_item_visible(theme_editor); - } else { theme_editor->set_process(false); if (theme_editor->is_visible_in_tree()) editor->hide_bottom_panel(); + button->hide(); } } From 2514594a4574ded407efdeb8688202423eb203e4 Mon Sep 17 00:00:00 2001 From: "thomas.herzog" Date: Wed, 22 May 2019 09:10:06 +0200 Subject: [PATCH 058/113] [GDNative] fix godot_variant_evaluate function Fixes #28984 (cherry picked from commit c1e733466bf8cd1dfe44787247c9ee54f4f85ae6) --- modules/gdnative/gdnative/variant.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gdnative/gdnative/variant.cpp b/modules/gdnative/gdnative/variant.cpp index 8f0d5a2db40..ac4d5a86b2b 100644 --- a/modules/gdnative/gdnative/variant.cpp +++ b/modules/gdnative/gdnative/variant.cpp @@ -518,7 +518,7 @@ void GDAPI godot_variant_evaluate(godot_variant_operator p_op, const godot_varia const Variant *a = (const Variant *)p_a; const Variant *b = (const Variant *)p_b; Variant *ret = (Variant *)r_ret; - Variant::evaluate(op, a, b, *ret, *r_valid); + Variant::evaluate(op, *a, *b, *ret, *r_valid); } #ifdef __cplusplus From f040f7c28339822293bbe253f5e656b696fd79ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20J=2E=20Est=C3=A9banez?= Date: Mon, 20 May 2019 22:35:31 +0200 Subject: [PATCH 059/113] Fix canvas zoom controls lazy position update After hiding/showing rulers, zoom controls were kept at the wrong position until an update happened because of other reason. (cherry picked from commit 4dd9efe75792dda34b80af443a7f5b1a510da889) --- editor/plugins/canvas_item_editor_plugin.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/editor/plugins/canvas_item_editor_plugin.cpp b/editor/plugins/canvas_item_editor_plugin.cpp index 3ecd8d48206..2abc96c5e60 100644 --- a/editor/plugins/canvas_item_editor_plugin.cpp +++ b/editor/plugins/canvas_item_editor_plugin.cpp @@ -3903,6 +3903,7 @@ void CanvasItemEditor::_popup_callback(int p_op) { show_rulers = !show_rulers; int idx = view_menu->get_popup()->get_item_index(SHOW_RULERS); view_menu->get_popup()->set_item_checked(idx, show_rulers); + _update_scrollbars(); viewport->update(); } break; case SHOW_GUIDES: { @@ -4381,6 +4382,7 @@ Dictionary CanvasItemEditor::get_state() const { void CanvasItemEditor::set_state(const Dictionary &p_state) { + bool update_scrollbars = false; Dictionary state = p_state; if (state.has("zoom")) { zoom = p_state["zoom"]; @@ -4389,7 +4391,7 @@ void CanvasItemEditor::set_state(const Dictionary &p_state) { if (state.has("ofs")) { view_offset = p_state["ofs"]; previous_update_view_offset = view_offset; - _update_scrollbars(); + update_scrollbars = true; } if (state.has("grid_offset")) { @@ -4477,6 +4479,7 @@ void CanvasItemEditor::set_state(const Dictionary &p_state) { show_rulers = state["show_rulers"]; int idx = view_menu->get_popup()->get_item_index(SHOW_RULERS); view_menu->get_popup()->set_item_checked(idx, show_rulers); + update_scrollbars = true; } if (state.has("show_guides")) { @@ -4521,6 +4524,9 @@ void CanvasItemEditor::set_state(const Dictionary &p_state) { skeleton_menu->get_popup()->set_item_checked(idx, skeleton_show_bones); } + if (update_scrollbars) { + _update_scrollbars(); + } viewport->update(); } From 340f542e4919dbd8c17bb6bc87a08dfc57af73ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20J=2E=20Est=C3=A9banez?= Date: Mon, 20 May 2019 22:36:34 +0200 Subject: [PATCH 060/113] Remove ghost of canvas zoom controls during onion skinning (cherry picked from commit 3b17e6de3398dced61bc3f5b5b2bdde67f2593c4) --- editor/plugins/animation_player_editor_plugin.cpp | 1 + editor/plugins/canvas_item_editor_plugin.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/editor/plugins/animation_player_editor_plugin.cpp b/editor/plugins/animation_player_editor_plugin.cpp index 41f35c3bed0..a863bf8c34d 100644 --- a/editor/plugins/animation_player_editor_plugin.cpp +++ b/editor/plugins/animation_player_editor_plugin.cpp @@ -1430,6 +1430,7 @@ void AnimationPlayerEditor::_prepare_onion_layers_2() { new_state["show_rulers"] = false; new_state["show_guides"] = false; new_state["show_helpers"] = false; + new_state["show_zoom_control"] = false; // TODO: Save/restore only affected entries CanvasItemEditor::get_singleton()->set_state(new_state); } diff --git a/editor/plugins/canvas_item_editor_plugin.cpp b/editor/plugins/canvas_item_editor_plugin.cpp index 2abc96c5e60..30c78722490 100644 --- a/editor/plugins/canvas_item_editor_plugin.cpp +++ b/editor/plugins/canvas_item_editor_plugin.cpp @@ -4372,6 +4372,7 @@ Dictionary CanvasItemEditor::get_state() const { state["show_rulers"] = show_rulers; state["show_guides"] = show_guides; state["show_helpers"] = show_helpers; + state["show_zoom_control"] = zoom_hb->is_visible(); state["show_edit_locks"] = show_edit_locks; state["snap_rotation"] = snap_rotation; state["snap_relative"] = snap_relative; @@ -4500,6 +4501,11 @@ void CanvasItemEditor::set_state(const Dictionary &p_state) { view_menu->get_popup()->set_item_checked(idx, show_edit_locks); } + if (state.has("show_zoom_control")) { + // This one is not user-controllable, but instrumentable + zoom_hb->set_visible(state["show_zoom_control"]); + } + if (state.has("snap_rotation")) { snap_rotation = state["snap_rotation"]; int idx = snap_config_menu->get_popup()->get_item_index(SNAP_USE_ROTATION); From aacefcb7dc7ce7058827427d6279fc98cd6a6fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20J=2E=20Est=C3=A9banez?= Date: Wed, 22 May 2019 20:40:19 +0200 Subject: [PATCH 061/113] Fix 2D bones ignored by onion skinning Fixes #27819. (cherry picked from commit 88153fbb6109f897ec3eac2180cdb757502aebb4) --- editor/plugins/animation_player_editor_plugin.cpp | 2 +- scene/2d/skeleton_2d.h | 6 ++++++ scene/animation/animation_player.cpp | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/editor/plugins/animation_player_editor_plugin.cpp b/editor/plugins/animation_player_editor_plugin.cpp index a863bf8c34d..0d49fdbb907 100644 --- a/editor/plugins/animation_player_editor_plugin.cpp +++ b/editor/plugins/animation_player_editor_plugin.cpp @@ -1483,7 +1483,7 @@ void AnimationPlayerEditor::_prepare_onion_layers_2() { if (valid) { player->seek(pos, true); get_tree()->flush_transform_notifications(); // Needed for transforms of Spatials - values_backup.update_skeletons(); // Needed for Skeletons + values_backup.update_skeletons(); // Needed for Skeletons (2D & 3D) VS::get_singleton()->viewport_set_active(onion.captures[cidx], true); VS::get_singleton()->viewport_set_parent_viewport(root_vp, onion.captures[cidx]); diff --git a/scene/2d/skeleton_2d.h b/scene/2d/skeleton_2d.h index cf9877e6f8c..d24c0a15618 100644 --- a/scene/2d/skeleton_2d.h +++ b/scene/2d/skeleton_2d.h @@ -39,6 +39,9 @@ class Bone2D : public Node2D { GDCLASS(Bone2D, Node2D) friend class Skeleton2D; +#ifdef TOOLS_ENABLED + friend class AnimatedValuesBackup; +#endif Bone2D *parent_bone; Skeleton2D *skeleton; @@ -71,6 +74,9 @@ class Skeleton2D : public Node2D { GDCLASS(Skeleton2D, Node2D); friend class Bone2D; +#ifdef TOOLS_ENABLED + friend class AnimatedValuesBackup; +#endif struct Bone { bool operator<(const Bone &p_bone) const { diff --git a/scene/animation/animation_player.cpp b/scene/animation/animation_player.cpp index 016db15b731..558231cea9b 100644 --- a/scene/animation/animation_player.cpp +++ b/scene/animation/animation_player.cpp @@ -35,11 +35,20 @@ #include "scene/scene_string_names.h" #include "servers/audio/audio_stream.h" #ifdef TOOLS_ENABLED +#include "scene/2d/skeleton_2d.h" + void AnimatedValuesBackup::update_skeletons() { for (int i = 0; i < entries.size(); i++) { if (entries[i].bone_idx != -1) { + // 3D bone Object::cast_to(entries[i].object)->notification(Skeleton::NOTIFICATION_UPDATE_SKELETON); + } else { + Bone2D *bone = Object::cast_to(entries[i].object); + if (bone && bone->skeleton) { + // 2D bone + bone->skeleton->_update_transform(); + } } } } From 7431d3c3c00ac2fef82949abf88920f55745f8b8 Mon Sep 17 00:00:00 2001 From: Tomasz Chabora Date: Thu, 23 May 2019 20:40:52 +0200 Subject: [PATCH 062/113] Don't modulate canvas on invisible color change (cherry picked from commit 5e9400b2e3f4d54d1c8f7227c01580276de2bbdb) --- scene/2d/canvas_modulate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/2d/canvas_modulate.cpp b/scene/2d/canvas_modulate.cpp index bd7bb97b039..009d6644629 100644 --- a/scene/2d/canvas_modulate.cpp +++ b/scene/2d/canvas_modulate.cpp @@ -70,7 +70,7 @@ void CanvasModulate::_bind_methods() { void CanvasModulate::set_color(const Color &p_color) { color = p_color; - if (is_inside_tree()) { + if (is_visible_in_tree()) { VS::get_singleton()->canvas_set_modulate(get_canvas(), color); } } From 4444bd1fcb53f7e127736100b3a2ec2d0a830cff Mon Sep 17 00:00:00 2001 From: WiggleWizard <1405402+WiggleWizard@users.noreply.github.com> Date: Thu, 23 May 2019 22:31:02 +0100 Subject: [PATCH 063/113] Fixed virtual method not showing up in autocomplete and docs (cherry picked from commit aebca30c43708a2ee6a1abc4d869eda88a3c27bb) --- doc/classes/EditorExportPlugin.xml | 6 ++++++ editor/editor_export.cpp | 1 + 2 files changed, 7 insertions(+) diff --git a/doc/classes/EditorExportPlugin.xml b/doc/classes/EditorExportPlugin.xml index 07c8eeccac5..76ff6df7939 100644 --- a/doc/classes/EditorExportPlugin.xml +++ b/doc/classes/EditorExportPlugin.xml @@ -21,6 +21,12 @@ + + + + + + diff --git a/editor/editor_export.cpp b/editor/editor_export.cpp index dc43faeff14..32f3c829864 100644 --- a/editor/editor_export.cpp +++ b/editor/editor_export.cpp @@ -611,6 +611,7 @@ void EditorExportPlugin::_bind_methods() { BIND_VMETHOD(MethodInfo("_export_file", PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::STRING, "type"), PropertyInfo(Variant::POOL_STRING_ARRAY, "features"))); BIND_VMETHOD(MethodInfo("_export_begin", PropertyInfo(Variant::POOL_STRING_ARRAY, "features"), PropertyInfo(Variant::BOOL, "is_debug"), PropertyInfo(Variant::STRING, "path"), PropertyInfo(Variant::INT, "flags"))); + BIND_VMETHOD(MethodInfo("_export_end")); } EditorExportPlugin::EditorExportPlugin() { From 6c71289432840819abe2a20780b50ba466b485c6 Mon Sep 17 00:00:00 2001 From: Geequlim Date: Fri, 24 May 2019 11:56:44 +0800 Subject: [PATCH 064/113] Fix parse error when extends from subclass of named classes (cherry picked from commit ebcfe284751deff4105c68b1d92511ace1a25665) --- modules/gdscript/gdscript_parser.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index 9ba15ec6a4a..0b2fe9c3766 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -5238,6 +5238,7 @@ void GDScriptParser::_determine_inheritance(ClassNode *p_class) { if (base_script.is_valid()) { String ident = base; + Ref find_subclass = base_script; for (int i = extend_iter; i < p_class->extends_class.size(); i++) { @@ -5247,7 +5248,7 @@ void GDScriptParser::_determine_inheritance(ClassNode *p_class) { if (base_script->get_subclasses().has(subclass)) { - base_script = base_script->get_subclasses()[subclass]; + find_subclass = base_script->get_subclasses()[subclass]; } else if (base_script->get_constants().has(subclass)) { Ref new_base_class = base_script->get_constants()[subclass]; @@ -5255,7 +5256,7 @@ void GDScriptParser::_determine_inheritance(ClassNode *p_class) { _set_error("Constant is not a class: " + ident, p_class->line); return; } - base_script = new_base_class; + find_subclass = new_base_class; } else { _set_error("Could not find subclass: " + ident, p_class->line); @@ -5263,7 +5264,7 @@ void GDScriptParser::_determine_inheritance(ClassNode *p_class) { } } - script = base_script; + script = find_subclass; } else if (!base_class) { From 6afcc724656c78f9f5925e4388b197d6b3b27c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20ANDR=C3=89-CHANG?= Date: Sun, 26 May 2019 16:33:51 +0100 Subject: [PATCH 065/113] Fix Segmentation fault and reduce memory consumption (cherry picked from commit a156b7976da6a7e0a6492ae2b336faa645a19da2) --- editor/project_manager.cpp | 24 ++++++------------------ editor/project_manager.h | 2 +- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/editor/project_manager.cpp b/editor/project_manager.cpp index 095ec891cd7..102d57f745f 100644 --- a/editor/project_manager.cpp +++ b/editor/project_manager.cpp @@ -1609,40 +1609,28 @@ void ProjectManager::_show_project(const String &p_path) { OS::get_singleton()->shell_open(String("file://") + p_path); } -void ProjectManager::_scan_dir(DirAccess *da, float pos, float total, List *r_projects) { - - List subdirs; +void ProjectManager::_scan_dir(const String &path, List *r_projects) { + DirAccess *da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); + da->change_dir(path); da->list_dir_begin(); String n = da->get_next(); while (n != String()) { if (da->current_is_dir() && !n.begins_with(".")) { - subdirs.push_front(n); + _scan_dir(da->get_current_dir().plus_file(n), r_projects); } else if (n == "project.godot") { r_projects->push_back(da->get_current_dir()); } n = da->get_next(); } da->list_dir_end(); - int m = 0; - for (List::Element *E = subdirs.front(); E; E = E->next()) { - - da->change_dir(E->get()); - - float slice = total / subdirs.size(); - _scan_dir(da, pos + slice * m, slice, r_projects); - da->change_dir(".."); - m++; - } + memdelete(da); } void ProjectManager::_scan_begin(const String &p_base) { print_line("Scanning projects at: " + p_base); List projects; - DirAccess *da = DirAccess::create(DirAccess::ACCESS_FILESYSTEM); - da->change_dir(p_base); - _scan_dir(da, 0, 1, &projects); - memdelete(da); + _scan_dir(p_base, &projects); print_line("Found " + itos(projects.size()) + " projects."); for (List::Element *E = projects.front(); E; E = E->next()) { diff --git a/editor/project_manager.h b/editor/project_manager.h index 1fdd7dbe063..d0b926886f1 100644 --- a/editor/project_manager.h +++ b/editor/project_manager.h @@ -102,7 +102,7 @@ class ProjectManager : public Control { void _on_project_created(const String &dir); void _on_projects_updated(); void _update_scroll_position(const String &dir); - void _scan_dir(DirAccess *da, float pos, float total, List *r_projects); + void _scan_dir(const String &path, List *r_projects); void _install_project(const String &p_zip_path, const String &p_title); From 88cccfb79614b2509151c9dd8f93c756e51015fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 28 May 2019 11:19:21 +0200 Subject: [PATCH 066/113] Print engine version to stdout when starting Godot Also include website URL and make it configurable via version.py together with the rest of the engine branding. Add mention to MIT license in --help output. (cherry picked from commit b10dd110e525ee7ce2dcf7721a51c93d782347de) --- main/main.cpp | 6 +++++- methods.py | 1 + version.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/main/main.cpp b/main/main.cpp index efcfb1a749e..88def7dccda 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -204,7 +204,8 @@ void finalize_physics() { void Main::print_help(const char *p_binary) { - print_line(String(VERSION_NAME) + " v" + get_full_version_string() + " - https://godotengine.org"); + print_line(String(VERSION_NAME) + " v" + get_full_version_string() + " - " + String(VERSION_WEBSITE)); + OS::get_singleton()->print("Free and open source software under the terms of the MIT license.\n"); OS::get_singleton()->print("(c) 2007-2019 Juan Linietsky, Ariel Manzur.\n"); OS::get_singleton()->print("(c) 2014-2019 Godot Engine contributors.\n"); OS::get_singleton()->print("\n"); @@ -1087,6 +1088,9 @@ error: Error Main::setup2(Thread::ID p_main_tid_override) { + // Print engine name and version + print_line(String(VERSION_NAME) + " v" + get_full_version_string() + " - " + String(VERSION_WEBSITE)); + if (p_main_tid_override) { Thread::_main_thread_id = p_main_tid_override; } diff --git a/methods.py b/methods.py index 11551a51d22..4f9a6ca470f 100644 --- a/methods.py +++ b/methods.py @@ -61,6 +61,7 @@ def update_version(module_version_string=""): f.write("#define VERSION_BUILD \"" + str(build_name) + "\"\n") f.write("#define VERSION_MODULE_CONFIG \"" + str(version.module_config) + module_version_string + "\"\n") f.write("#define VERSION_YEAR " + str(version.year) + "\n") + f.write("#define VERSION_WEBSITE \"" + str(version.website) + "\"\n") f.close() # NOTE: It is safe to generate this file here, since this is still executed serially diff --git a/version.py b/version.py index ae094ede520..f0958d0ba2b 100644 --- a/version.py +++ b/version.py @@ -6,3 +6,4 @@ patch = 2 status = "devel" module_config = "" year = 2019 +website = "https://godotengine.org" From 9591e2950aaf20a36df670619757ffa49b0f8e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 28 May 2019 12:40:39 +0200 Subject: [PATCH 067/113] Node::duplicate: Don't set name if original node is nameless In practice this only happens when duplicating a node which is not in the scene tree yet, as nameless nodes get assigned a generated name when added to the scene tree. Fixes #27319. (cherry picked from commit 0823ae7ae47989c31d64de222d6f6e4bdcea1962) --- scene/main/node.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scene/main/node.cpp b/scene/main/node.cpp index 93c5140a8cb..5ff22f27fb4 100644 --- a/scene/main/node.cpp +++ b/scene/main/node.cpp @@ -2063,7 +2063,9 @@ Node *Node::_duplicate(int p_flags, Map *r_duplimap) const } } - node->set_name(get_name()); + if (get_name() != String()) { + node->set_name(get_name()); + } #ifdef TOOLS_ENABLED if ((p_flags & DUPLICATE_FROM_EDITOR) && r_duplimap) From f11588e5a50b9a0177f8ae379d4bb70ffb64f18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Wed, 29 May 2019 17:28:22 +0200 Subject: [PATCH 068/113] Fix wrong property binding for NavigationPolygon::vertices (cherry picked from commit 586539d86771dc9cf2588e41d389b5b2be9d0b88) --- scene/2d/navigation_polygon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/2d/navigation_polygon.cpp b/scene/2d/navigation_polygon.cpp index 0f6af358bde..e389d5f98fc 100644 --- a/scene/2d/navigation_polygon.cpp +++ b/scene/2d/navigation_polygon.cpp @@ -312,7 +312,7 @@ void NavigationPolygon::_bind_methods() { ClassDB::bind_method(D_METHOD("_set_outlines", "outlines"), &NavigationPolygon::_set_outlines); ClassDB::bind_method(D_METHOD("_get_outlines"), &NavigationPolygon::_get_outlines); - ADD_PROPERTY(PropertyInfo(Variant::POOL_VECTOR3_ARRAY, "vertices", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "set_vertices", "get_vertices"); + ADD_PROPERTY(PropertyInfo(Variant::POOL_VECTOR2_ARRAY, "vertices", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "set_vertices", "get_vertices"); ADD_PROPERTY(PropertyInfo(Variant::ARRAY, "polygons", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_polygons", "_get_polygons"); ADD_PROPERTY(PropertyInfo(Variant::ARRAY, "outlines", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NOEDITOR | PROPERTY_USAGE_INTERNAL), "_set_outlines", "_get_outlines"); } From e66122e0bf049abfff98f59e8a39f1924b5358e5 Mon Sep 17 00:00:00 2001 From: Tomasz Chabora Date: Sat, 1 Jun 2019 14:42:13 +0200 Subject: [PATCH 069/113] Fixed get_item_at_position being weird (cherry picked from commit 59f403de0bc815d19e59b0445c1a111018a9d339) --- scene/gui/item_list.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/gui/item_list.cpp b/scene/gui/item_list.cpp index 026374ded1d..69fb09f64b8 100644 --- a/scene/gui/item_list.cpp +++ b/scene/gui/item_list.cpp @@ -1243,7 +1243,7 @@ int ItemList::get_item_at_position(const Point2 &p_pos, bool p_exact) const { Rect2 rc = items[i].rect_cache; if (i % current_columns == current_columns - 1) { - rc.size.width = get_size().width; //not right but works + rc.size.width = get_size().width - rc.position.x; //make sure you can still select the last item when clicking past the column } if (rc.has_point(pos)) { From 4d6517fad358abd83ac30559bc939304acf0b331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Mon, 3 Jun 2019 11:18:09 +0200 Subject: [PATCH 070/113] Bind missing FLAG_DISABLE_Z enum value in ParticlesMaterial Fixes #29419. (cherry picked from commit a3462c97adc63461b12462d57c75a67ce54cecfd) --- doc/classes/ParticlesMaterial.xml | 7 ++++++- scene/resources/particles_material.cpp | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/classes/ParticlesMaterial.xml b/doc/classes/ParticlesMaterial.xml index 745bd3b32c0..691043ec7b2 100644 --- a/doc/classes/ParticlesMaterial.xml +++ b/doc/classes/ParticlesMaterial.xml @@ -214,14 +214,19 @@ Use with [method set_param], [method set_param_randomness], and [method set_param_texture] to set animation offset properties. + Represents the size of the [enum Parameter] enum. Use with [method set_flag] to set [member flag_align_y]. - Use with [method set_flag] to set [member flag_rotate_y] + Use with [method set_flag] to set [member flag_rotate_y]. + + + Use with [method set_flag] to set [member flag_disable_z]. + Represents the size of the [enum Flags] enum. All particles will be emitted from a single point. diff --git a/scene/resources/particles_material.cpp b/scene/resources/particles_material.cpp index 7534891edb9..aac6594846e 100644 --- a/scene/resources/particles_material.cpp +++ b/scene/resources/particles_material.cpp @@ -1187,6 +1187,7 @@ void ParticlesMaterial::_bind_methods() { BIND_ENUM_CONSTANT(FLAG_ALIGN_Y_TO_VELOCITY); BIND_ENUM_CONSTANT(FLAG_ROTATE_Y); + BIND_ENUM_CONSTANT(FLAG_DISABLE_Z); BIND_ENUM_CONSTANT(FLAG_MAX); BIND_ENUM_CONSTANT(EMISSION_SHAPE_POINT); From 22abeff592957b0fab974eaaa9bd0741f13a6ef4 Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Mon, 3 Jun 2019 16:54:41 +0300 Subject: [PATCH 071/113] Escape node names when saving to .tscn Fixes #29401 (cherry picked from commit cc004485bc994c39065a8a4a6cff861a383f6fb0) --- scene/resources/resource_format_text.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scene/resources/resource_format_text.cpp b/scene/resources/resource_format_text.cpp index b6efca9acc3..f27f9b82150 100644 --- a/scene/resources/resource_format_text.cpp +++ b/scene/resources/resource_format_text.cpp @@ -1664,15 +1664,15 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const RES &p_r Vector groups = state->get_node_groups(i); String header = "[node"; - header += " name=\"" + String(name) + "\""; + header += " name=\"" + String(name).c_escape() + "\""; if (type != StringName()) { header += " type=\"" + String(type) + "\""; } if (path != NodePath()) { - header += " parent=\"" + String(path.simplified()) + "\""; + header += " parent=\"" + String(path.simplified()).c_escape() + "\""; } if (owner != NodePath() && owner != NodePath(".")) { - header += " owner=\"" + String(owner.simplified()) + "\""; + header += " owner=\"" + String(owner.simplified()).c_escape() + "\""; } if (index >= 0) { header += " index=\"" + itos(index) + "\""; From 02ccd4d78caecc082abd92d233298bdd1d8d5488 Mon Sep 17 00:00:00 2001 From: Michael Alexsander Silva Dias Date: Mon, 3 Jun 2019 20:01:27 -0300 Subject: [PATCH 072/113] Remember selected subtile when updating the TileMap editor Fixes #29444. (cherry picked from commit 0a03ed7544893038f17445df536366ee863f2f36) --- editor/plugins/tile_map_editor_plugin.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/editor/plugins/tile_map_editor_plugin.cpp b/editor/plugins/tile_map_editor_plugin.cpp index fa0de1077f8..b5b18374550 100644 --- a/editor/plugins/tile_map_editor_plugin.cpp +++ b/editor/plugins/tile_map_editor_plugin.cpp @@ -363,6 +363,8 @@ void TileMapEditor::_update_palette() { // Update the palette Vector selected = get_selected_tiles(); + int selected_single = palette->get_current(); + int selected_manual = manual_palette->get_current(); palette->clear(); manual_palette->clear(); manual_palette->hide(); @@ -511,9 +513,10 @@ void TileMapEditor::_update_palette() { if (manual_palette->get_item_count() > 0) { // Only show the manual palette if at least tile exists in it - int selected2 = manual_palette->get_current(); - if (selected2 == -1) selected2 = 0; - manual_palette->set_current(selected2); + if (selected_manual == -1 || selected_single != palette->get_current()) + selected_manual = 0; + if (selected_manual < manual_palette->get_item_count()) + manual_palette->set_current(selected_manual); manual_palette->show(); } From ef6d414e6e25c190a30e2f58c5822cb56ae4fbd3 Mon Sep 17 00:00:00 2001 From: Hein-Pieter van Braam-Stewart Date: Tue, 4 Jun 2019 05:36:23 +0200 Subject: [PATCH 073/113] Fix moving and renaming files This commit fixes several issues related to moving scenes and resources in an open project. * Don't try to reload scenes while not all scenes are updated yet. * Don't use the UndoRedo system to update non-user initiated editor state. * Resave scenes after moving files and updating resource path(s). (cherry picked from commit 0ac7715a27abc9a679f9537cc16ebb7e3d3bfea4) --- editor/editor_data.cpp | 1 + editor/editor_node.cpp | 44 +++++++++++++++++++++++++++++--------- editor/editor_node.h | 11 ++++++++-- editor/filesystem_dock.cpp | 31 ++++++++++++++++++++++++++- editor/filesystem_dock.h | 1 + 5 files changed, 75 insertions(+), 13 deletions(-) diff --git a/editor/editor_data.cpp b/editor/editor_data.cpp index f61a8310159..38f30df1694 100644 --- a/editor/editor_data.cpp +++ b/editor/editor_data.cpp @@ -560,6 +560,7 @@ void EditorData::move_edited_scene_index(int p_idx, int p_to_idx) { ERR_FAIL_INDEX(p_to_idx, edited_scene.size()); SWAP(edited_scene.write[p_idx], edited_scene.write[p_to_idx]); } + void EditorData::remove_scene(int p_idx) { ERR_FAIL_INDEX(p_idx, edited_scene.size()); if (edited_scene[p_idx].root) { diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index edcd7d64781..47eeef55ac8 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -504,6 +504,7 @@ void EditorNode::_fs_changed() { void EditorNode::_resources_reimported(const Vector &p_resources) { List scenes; //will load later + int current_tab = scene_tabs->get_current_tab(); for (int i = 0; i < p_resources.size(); i++) { String file_type = ResourceLoader::get_resource_type(p_resources[i]); @@ -526,6 +527,8 @@ void EditorNode::_resources_reimported(const Vector &p_resources) { for (List::Element *E = scenes.front(); E; E = E->next()) { reload_scene(E->get()); } + + scene_tabs->set_current_tab(current_tab); } void EditorNode::_sources_changed(bool p_exist) { @@ -1189,6 +1192,17 @@ void EditorNode::save_all_scenes() { _save_all_scenes(); } +void EditorNode::save_scene_list(Vector p_scene_filenames) { + + for (int i = 0; i < editor_data.get_edited_scene_count(); i++) { + Node *scene = editor_data.get_edited_scene_root(i); + + if (scene && (p_scene_filenames.find(scene->get_filename()) >= 0)) { + _save_scene(scene->get_filename(), i); + } + } +} + void EditorNode::restart_editor() { exiting = true; @@ -2744,7 +2758,7 @@ bool EditorNode::is_addon_plugin_enabled(const String &p_addon) const { return plugin_addons.has(p_addon); } -void EditorNode::_remove_edited_scene() { +void EditorNode::_remove_edited_scene(bool p_change_tab) { int new_index = editor_data.get_edited_scene(); int old_index = new_index; @@ -2760,18 +2774,19 @@ void EditorNode::_remove_edited_scene() { if (editor_data.get_scene_path(old_index) != String()) { ScriptEditor::get_singleton()->close_builtin_scripts_from_scene(editor_data.get_scene_path(old_index)); } - _scene_tab_changed(new_index); + + if (p_change_tab) _scene_tab_changed(new_index); editor_data.remove_scene(old_index); editor_data.get_undo_redo().clear_history(false); _update_title(); _update_scene_tabs(); } -void EditorNode::_remove_scene(int index) { +void EditorNode::_remove_scene(int index, bool p_change_tab) { if (editor_data.get_edited_scene() == index) { //Scene to remove is current scene - _remove_edited_scene(); + _remove_edited_scene(p_change_tab); } else { //Scene to remove is not active scene editor_data.remove_scene(index); @@ -4023,6 +4038,14 @@ bool EditorNode::has_scenes_in_session() { return !scenes.empty(); } +int EditorNode::get_current_tab() { + return scene_tabs->get_current_tab(); +} + +void EditorNode::set_current_tab(int p_tab) { + scene_tabs->set_current_tab(p_tab); +} + void EditorNode::_update_layouts_menu() { editor_layouts->clear(); @@ -4612,8 +4635,7 @@ void EditorNode::reload_scene(const String &p_path) { if (scene_idx == -1) { if (get_edited_scene()) { - //scene is not open, so at it might be instanced, just refresh, set tab to itself and it will reload - set_current_scene(current_tab); + //scene is not open, so at it might be instanced. We'll refresh the whole scene later. editor_data.get_undo_redo().clear_history(); } return; @@ -4623,17 +4645,19 @@ void EditorNode::reload_scene(const String &p_path) { editor_data.apply_changes_in_editors(); _set_scene_metadata(p_path); } - //remove scene - _remove_scene(scene_idx); - //reload scene + //remove scene + _remove_scene(scene_idx, false); + + //reload scene load_scene(p_path, true, false, true, true); + //adjust index so tab is back a the previous position editor_data.move_edited_scene_to_index(scene_idx); get_undo_redo()->clear_history(); + //recover the tab scene_tabs->set_current_tab(current_tab); - _scene_tab_changed(current_tab); } int EditorNode::plugin_init_callback_count = 0; diff --git a/editor/editor_node.h b/editor/editor_node.h index e853703cab2..85b036710b2 100644 --- a/editor/editor_node.h +++ b/editor/editor_node.h @@ -499,8 +499,8 @@ private: static void _editor_file_dialog_unregister(EditorFileDialog *p_dialog); void _cleanup_scene(); - void _remove_edited_scene(); - void _remove_scene(int index); + void _remove_edited_scene(bool p_change_tab = true); + void _remove_scene(int index, bool p_change_tab = true); bool _find_and_save_resource(RES p_res, Map &processed, int32_t flags); bool _find_and_save_edited_subresources(Object *obj, Map &processed, int32_t flags); void _save_edited_subresources(Node *scene, Map &processed, int32_t flags); @@ -615,6 +615,12 @@ protected: void _notification(int p_what); static void _bind_methods(); +protected: + friend class FileSystemDock; + + int get_current_tab(); + void set_current_tab(int p_tab); + public: bool call_build(); @@ -786,6 +792,7 @@ public: void remove_tool_menu_item(const String &p_name); void save_all_scenes(); + void save_scene_list(Vector p_scene_filenames); void restart_editor(); void dim_editor(bool p_dimming); diff --git a/editor/filesystem_dock.cpp b/editor/filesystem_dock.cpp index a22961d1ff6..26e33d76916 100644 --- a/editor/filesystem_dock.cpp +++ b/editor/filesystem_dock.cpp @@ -1167,6 +1167,21 @@ void FileSystemDock::_update_favorites_list_after_move(const Map EditorSettings::get_singleton()->set_favorites(new_favorites); } +void FileSystemDock::_save_scenes_after_move(const Map &p_renames) const { + Vector remaps; + _find_remaps(EditorFileSystem::get_singleton()->get_filesystem(), p_renames, remaps); + Vector new_filenames; + + for (int i = 0; i < remaps.size(); ++i) { + String file = p_renames.has(remaps[i]) ? p_renames[remaps[i]] : remaps[i]; + if (ResourceLoader::get_resource_type(file) == "PackedScene") { + new_filenames.push_back(file); + } + } + + editor->save_scene_list(new_filenames); +} + void FileSystemDock::_make_dir_confirm() { String dir_name = make_dir_dialog_text->get_text().strip_edges(); @@ -1241,14 +1256,21 @@ void FileSystemDock::_rename_operation_confirm() { Map file_renames; Map folder_renames; _try_move_item(to_rename, new_path, file_renames, folder_renames); + + int current_tab = editor->get_current_tab(); + _update_dependencies_after_move(file_renames); _update_resource_paths_after_move(file_renames); _update_project_settings_after_move(file_renames); _update_favorites_list_after_move(file_renames, folder_renames); - //Rescan everything + editor->set_current_tab(current_tab); + print_verbose("FileSystem: calling rescan."); _rescan(); + + print_verbose("FileSystem: saving moved scenes."); + _save_scenes_after_move(file_renames); } void FileSystemDock::_duplicate_operation_confirm() { @@ -1334,13 +1356,20 @@ void FileSystemDock::_move_operation_confirm(const String &p_to_path, bool overw } if (is_moved) { + int current_tab = editor->get_current_tab(); + _update_dependencies_after_move(file_renames); _update_resource_paths_after_move(file_renames); _update_project_settings_after_move(file_renames); _update_favorites_list_after_move(file_renames, folder_renames); + editor->set_current_tab(current_tab); + print_verbose("FileSystem: calling rescan."); _rescan(); + + print_verbose("FileSystem: saving moved scenes."); + _save_scenes_after_move(file_renames); } } diff --git a/editor/filesystem_dock.h b/editor/filesystem_dock.h index ab0dec6f5ec..35087940b07 100644 --- a/editor/filesystem_dock.h +++ b/editor/filesystem_dock.h @@ -201,6 +201,7 @@ private: void _try_duplicate_item(const FileOrFolder &p_item, const String &p_new_path) const; void _update_dependencies_after_move(const Map &p_renames) const; void _update_resource_paths_after_move(const Map &p_renames) const; + void _save_scenes_after_move(const Map &p_renames) const; void _update_favorites_list_after_move(const Map &p_files_renames, const Map &p_folders_renames) const; void _update_project_settings_after_move(const Map &p_folders_renames) const; From 1804a726b4907506ba679ac16e6349902a5cade2 Mon Sep 17 00:00:00 2001 From: Michael Alexsander Silva Dias Date: Tue, 4 Jun 2019 01:44:38 -0300 Subject: [PATCH 074/113] Fix error spam when typing in the TileMap editor's search box while having no results (cherry picked from commit 412a0b73e9291b10a89780014c66947a2588ea85) --- editor/plugins/tile_map_editor_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editor/plugins/tile_map_editor_plugin.cpp b/editor/plugins/tile_map_editor_plugin.cpp index b5b18374550..bb3b829dc0d 100644 --- a/editor/plugins/tile_map_editor_plugin.cpp +++ b/editor/plugins/tile_map_editor_plugin.cpp @@ -472,7 +472,7 @@ void TileMapEditor::_update_palette() { if (selected.get(0) != TileMap::INVALID_CELL) { set_selected_tiles(selected); sel_tile = selected.get(Math::rand() % selected.size()); - } else { + } else if (palette->get_item_count() > 0) { palette->select(0); } From cb394082ef7609e48cc8cfde67ab1e21926f32ca Mon Sep 17 00:00:00 2001 From: qarmin Date: Wed, 5 Jun 2019 09:34:16 +0200 Subject: [PATCH 075/113] Fix memory leak after closing files (cherry picked from commit 53c0a29fb05343e979fda7faf4fde35020e0fefe) --- editor/plugins/asset_library_editor_plugin.cpp | 4 ++++ platform/uwp/export/export.cpp | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/editor/plugins/asset_library_editor_plugin.cpp b/editor/plugins/asset_library_editor_plugin.cpp index 6d2cdfc5839..e53ee83dc34 100644 --- a/editor/plugins/asset_library_editor_plugin.cpp +++ b/editor/plugins/asset_library_editor_plugin.cpp @@ -726,6 +726,7 @@ void EditorAssetLibrary::_image_update(bool use_cache, bool final, const PoolByt image_data = cached_data; file->close(); + memdelete(file); } } @@ -800,6 +801,7 @@ void EditorAssetLibrary::_image_request_completed(int p_status, int p_code, cons if (file) { file->store_line(new_etag); file->close(); + memdelete(file); } int len = p_data.size(); @@ -809,6 +811,7 @@ void EditorAssetLibrary::_image_request_completed(int p_status, int p_code, cons file->store_32(len); file->store_buffer(r.ptr(), len); file->close(); + memdelete(file); } break; @@ -848,6 +851,7 @@ void EditorAssetLibrary::_update_image_queue() { if (file) { headers.push_back("If-None-Match: " + file->get_line()); file->close(); + memdelete(file); } } diff --git a/platform/uwp/export/export.cpp b/platform/uwp/export/export.cpp index a0ab398f892..dbccfc904da 100644 --- a/platform/uwp/export/export.cpp +++ b/platform/uwp/export/export.cpp @@ -241,7 +241,6 @@ void AppxPackager::make_block_map() { tmp_file->close(); memdelete(tmp_file); - tmp_file = NULL; } String AppxPackager::content_type(String p_extension) { @@ -291,7 +290,6 @@ void AppxPackager::make_content_types() { tmp_file->close(); memdelete(tmp_file); - tmp_file = NULL; } Vector AppxPackager::make_file_header(FileMeta p_file_meta) { @@ -602,7 +600,6 @@ void AppxPackager::finish() { blockmap_file->close(); memdelete(blockmap_file); - blockmap_file = NULL; // Add content types EditorNode::progress_task_step("export", "Setting content types...", 5); @@ -618,7 +615,6 @@ void AppxPackager::finish() { types_file->close(); memdelete(types_file); - types_file = NULL; // Pre-process central directory before signing for (int i = 0; i < file_metadata.size(); i++) { From dce7311aa54a4fd862e56c73965193484d3e0286 Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Wed, 5 Jun 2019 11:43:32 +0300 Subject: [PATCH 076/113] =?UTF-8?q?Fix=20third=20and=20fourth=20level=20(A?= =?UTF-8?q?ltGr=20/=20=E2=8C=A5=20modifier)=20keys=20input=20in=20LineEdit?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 63d8dc09208152f76cb47eb191ce04377927a9fc) --- scene/gui/line_edit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/gui/line_edit.cpp b/scene/gui/line_edit.cpp index 290548f0168..a125e5738ea 100644 --- a/scene/gui/line_edit.cpp +++ b/scene/gui/line_edit.cpp @@ -505,7 +505,7 @@ void LineEdit::_gui_input(Ref p_event) { if (handled) { accept_event(); - } else if (!k->get_alt() && !k->get_command()) { + } else if (!k->get_command()) { if (k->get_unicode() >= 32 && k->get_scancode() != KEY_DELETE) { if (editable) { From d871ac670dc7769920fe6cd84966773388c616f6 Mon Sep 17 00:00:00 2001 From: Hugo Locurcio Date: Wed, 5 Jun 2019 16:44:20 +0200 Subject: [PATCH 077/113] Use uppercase for all hexadecimal constants in the editor help (cherry picked from commit 6516098d1216be74d6caba73d56eaaf7d30812fd) --- editor/editor_help.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/editor/editor_help.cpp b/editor/editor_help.cpp index 1842d5e1e92..bedffbcdb5f 100644 --- a/editor/editor_help.cpp +++ b/editor/editor_help.cpp @@ -175,8 +175,9 @@ String EditorHelp::_fix_constant(const String &p_constant) const { if (p_constant.strip_edges() == "2147483647") { return "0x7FFFFFFF"; } + if (p_constant.strip_edges() == "1048575") { - return "0xfffff"; + return "0xFFFFF"; } return p_constant; From 8423d98e1c5df6a2b146cc8d8a9c1e1f25d32768 Mon Sep 17 00:00:00 2001 From: qarmin Date: Thu, 6 Jun 2019 12:59:29 +0200 Subject: [PATCH 078/113] Fix crash with get_keywords_color (cherry picked from commit 8d993f635776cb5151bba159939700e490576593) --- scene/gui/text_edit.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scene/gui/text_edit.cpp b/scene/gui/text_edit.cpp index 631ddaf7de5..986f2e5d4cd 100644 --- a/scene/gui/text_edit.cpp +++ b/scene/gui/text_edit.cpp @@ -4457,6 +4457,8 @@ bool TextEdit::has_keyword_color(String p_keyword) const { } Color TextEdit::get_keyword_color(String p_keyword) const { + + ERR_FAIL_COND_V(!keywords.has(p_keyword), Color()) return keywords[p_keyword]; } From 1f5f2d3ce8cc0cdfee55cdbef1397b39ee772d3d Mon Sep 17 00:00:00 2001 From: qarmin Date: Thu, 6 Jun 2019 14:33:04 +0200 Subject: [PATCH 079/113] Fix ARVRAnchor and ARVRController error spam (cherry picked from commit 3de3f87657be62965602c83e670c5b589cdfee3c) --- scene/3d/arvr_nodes.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scene/3d/arvr_nodes.cpp b/scene/3d/arvr_nodes.cpp index 17b698c1b87..c72ccf042a8 100644 --- a/scene/3d/arvr_nodes.cpp +++ b/scene/3d/arvr_nodes.cpp @@ -376,7 +376,7 @@ String ARVRController::get_configuration_warning() const { }; ARVRController::ARVRController() { - controller_id = 0; + controller_id = 1; is_active = true; button_states = 0; }; @@ -502,7 +502,7 @@ Plane ARVRAnchor::get_plane() const { }; ARVRAnchor::ARVRAnchor() { - anchor_id = 0; + anchor_id = 1; is_active = true; }; From 03a1570a745529e72348a5ebd88c6d773913a0a9 Mon Sep 17 00:00:00 2001 From: qarmin Date: Thu, 6 Jun 2019 21:09:37 +0200 Subject: [PATCH 080/113] Fix 2D Line crash (cherry picked from commit 2621131549164d0a6107d7c957a792c1707d6ce4) --- scene/2d/line_2d.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/scene/2d/line_2d.cpp b/scene/2d/line_2d.cpp index 73692e05356..b4d6ccb9e8d 100644 --- a/scene/2d/line_2d.cpp +++ b/scene/2d/line_2d.cpp @@ -105,6 +105,7 @@ void Line2D::set_point_position(int i, Vector2 pos) { } Vector2 Line2D::get_point_position(int i) const { + ERR_FAIL_INDEX_V(i, _points.size(), Vector2()) return _points.get(i); } From df2a809aae02ca3338c43f23b8fca5d06a936d58 Mon Sep 17 00:00:00 2001 From: SamSindt Date: Thu, 6 Jun 2019 12:49:48 -0700 Subject: [PATCH 081/113] Reversed operator in SortLifetime fixing #29440 (cherry picked from commit 6a23eff748b671a0a34962356d8ac36626b90f5b) --- scene/2d/cpu_particles_2d.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/2d/cpu_particles_2d.h b/scene/2d/cpu_particles_2d.h index 2ec7badd76c..ccba576cb13 100644 --- a/scene/2d/cpu_particles_2d.h +++ b/scene/2d/cpu_particles_2d.h @@ -116,7 +116,7 @@ private: const Particle *particles; bool operator()(int p_a, int p_b) const { - return particles[p_a].time < particles[p_b].time; + return particles[p_a].time > particles[p_b].time; } }; From 1d2aaa447e31770acad65f2934a43f8f87b2750d Mon Sep 17 00:00:00 2001 From: qarmin Date: Fri, 7 Jun 2019 12:36:11 +0200 Subject: [PATCH 082/113] Fix RichLabelText::_process_line crash (cherry picked from commit 685451967c1dfe7b8820b74e53ad562a811be1d2) --- scene/gui/rich_text_label.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/gui/rich_text_label.cpp b/scene/gui/rich_text_label.cpp index be7c7d25d87..e649de2e1ef 100644 --- a/scene/gui/rich_text_label.cpp +++ b/scene/gui/rich_text_label.cpp @@ -592,7 +592,7 @@ int RichTextLabel::_process_line(ItemFrame *p_frame, const Vector2 &p_ofs, int & //assign actual widths for (int i = 0; i < table->columns.size(); i++) { table->columns.write[i].width = table->columns[i].min_width; - if (table->columns[i].expand) + if (table->columns[i].expand && total_ratio > 0) table->columns.write[i].width += table->columns[i].expand_ratio * remaining_width / total_ratio; table->total_width += table->columns[i].width + hseparation; } From b56fa430907a07ac812ab27e88f1de9bbb981bea Mon Sep 17 00:00:00 2001 From: Toshiwo Date: Fri, 7 Jun 2019 11:28:50 -0500 Subject: [PATCH 083/113] Fix Vector3 slerp method normalizing cross product (cherry picked from commit 416a7d06ded6f19a7c44c793bc536e338a48eec8) --- core/math/vector3.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/math/vector3.h b/core/math/vector3.h index e9074c5bd42..683bd47c716 100644 --- a/core/math/vector3.h +++ b/core/math/vector3.h @@ -223,7 +223,7 @@ Vector3 Vector3::slerp(const Vector3 &p_b, real_t p_t) const { #endif real_t theta = angle_to(p_b); - return rotated(cross(p_b), theta * p_t); + return rotated(cross(p_b).normalized(), theta * p_t); } real_t Vector3::distance_to(const Vector3 &p_b) const { From 656f23fde92468d94915be851a0e8fa59e110c2d Mon Sep 17 00:00:00 2001 From: Toshiwo Date: Fri, 5 Jul 2019 11:17:52 -0500 Subject: [PATCH 084/113] when doing Vector3 slerp it is not necessary to have it normalized. (cherry picked from commit 3aff6451143247b21c1dfb5f47d0f23cae79e8dd) --- core/math/vector3.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/math/vector3.h b/core/math/vector3.h index 683bd47c716..68f3ac3ee74 100644 --- a/core/math/vector3.h +++ b/core/math/vector3.h @@ -218,10 +218,6 @@ Vector3 Vector3::linear_interpolate(const Vector3 &p_b, real_t p_t) const { } Vector3 Vector3::slerp(const Vector3 &p_b, real_t p_t) const { -#ifdef MATH_CHECKS - ERR_FAIL_COND_V(!is_normalized(), Vector3()); -#endif - real_t theta = angle_to(p_b); return rotated(cross(p_b).normalized(), theta * p_t); } From e23bde4358b1ccd52520b57806f8524b0b985359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 11 Jun 2019 09:20:42 +0200 Subject: [PATCH 085/113] Always pass script path to external editor Previously you had to set the Exec Flags manually, reading the documentation to find out what placeholders to use. As most editors should support having the file path passed as the last argument, we default to doing this if no custom {file} flag is defined. We also default the Exec Flags to "{file}" and the placeholder text gives some documentation. Fixes #29662. (cherry picked from commit 4a757a2d96655e46de8d45bab59eb27c7eaf5ad7) --- editor/plugins/script_editor_plugin.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/editor/plugins/script_editor_plugin.cpp b/editor/plugins/script_editor_plugin.cpp index 3bc363f3d83..c1d85a78fa8 100644 --- a/editor/plugins/script_editor_plugin.cpp +++ b/editor/plugins/script_editor_plugin.cpp @@ -1906,10 +1906,11 @@ bool ScriptEditor::edit(const RES &p_resource, int p_line, int p_col, bool p_gra String flags = EditorSettings::get_singleton()->get("text_editor/external/exec_flags"); List args; + bool has_file_flag = false; + String script_path = ProjectSettings::get_singleton()->globalize_path(p_resource->get_path()); if (flags.size()) { String project_path = ProjectSettings::get_singleton()->get_resource_path(); - String script_path = ProjectSettings::get_singleton()->globalize_path(p_resource->get_path()); flags = flags.replacen("{line}", itos(p_line > 0 ? p_line : 0)); flags = flags.replacen("{col}", itos(p_col)); @@ -1931,6 +1932,9 @@ bool ScriptEditor::edit(const RES &p_resource, int p_line, int p_col, bool p_gra } else if (flags[i] == '\0' || (!inside_quotes && flags[i] == ' ')) { String arg = flags.substr(from, num_chars); + if (arg.find("{file}") != -1) { + has_file_flag = true; + } // do path replacement here, else there will be issues with spaces and quotes arg = arg.replacen("{project}", project_path); @@ -1945,6 +1949,11 @@ bool ScriptEditor::edit(const RES &p_resource, int p_line, int p_col, bool p_gra } } + // Default to passing script path if no {file} flag is specified. + if (!has_file_flag) { + args.push_back(script_path); + } + Error err = OS::get_singleton()->execute(path, args, false); if (err == OK) return false; @@ -3331,7 +3340,8 @@ ScriptEditorPlugin::ScriptEditorPlugin(EditorNode *p_node) { EditorSettings::get_singleton()->add_property_hint(PropertyInfo(Variant::INT, "text_editor/open_scripts/list_script_names_as", PROPERTY_HINT_ENUM, "Name,Parent Directory And Name,Full Path")); EDITOR_DEF("text_editor/open_scripts/list_script_names_as", 0); EditorSettings::get_singleton()->add_property_hint(PropertyInfo(Variant::STRING, "text_editor/external/exec_path", PROPERTY_HINT_GLOBAL_FILE)); - EDITOR_DEF("text_editor/external/exec_flags", ""); + EDITOR_DEF("text_editor/external/exec_flags", "{file}"); + EditorSettings::get_singleton()->add_property_hint(PropertyInfo(Variant::STRING, "text_editor/external/exec_flags", PROPERTY_HINT_PLACEHOLDER_TEXT, "Call flags with placeholders: {project}, {file}, {col}, {line}.")); ED_SHORTCUT("script_editor/open_recent", TTR("Open Recent"), KEY_MASK_CMD | KEY_MASK_SHIFT | KEY_T); ED_SHORTCUT("script_editor/clear_recent", TTR("Clear Recent Files")); From 9e2f76f17dbb8e38104c07d7b5c64f15da0b31ed Mon Sep 17 00:00:00 2001 From: bruvzg <7645683+bruvzg@users.noreply.github.com> Date: Tue, 11 Jun 2019 12:07:48 +0300 Subject: [PATCH 086/113] [X11] Add window borderless state detection, fix borderless state restoration after exiting fullscreen. (cherry picked from commit 2a950f3a7c2a1903208059bdc3bdc190a70e44cc) --- platform/x11/os_x11.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/platform/x11/os_x11.cpp b/platform/x11/os_x11.cpp index ebd4b0cc4da..ced1f20a38b 100644 --- a/platform/x11/os_x11.cpp +++ b/platform/x11/os_x11.cpp @@ -1015,12 +1015,12 @@ void OS_X11::set_wm_fullscreen(bool p_enabled) { XFree(xsh); } - if (!p_enabled && !get_borderless_window()) { - // put decorations back if the window wasn't suppoesed to be borderless + if (!p_enabled) { + // put back or remove decorations according to the last set borderless state Hints hints; Atom property; hints.flags = 2; - hints.decorations = 1; + hints.decorations = current_videomode.borderless_window ? 0 : 1; property = XInternAtom(x11_display, "_MOTIF_WM_HINTS", True); XChangeProperty(x11_display, x11_window, property, property, 32, PropModeReplace, (unsigned char *)&hints, 5); } @@ -1509,7 +1509,7 @@ bool OS_X11::is_window_always_on_top() const { void OS_X11::set_borderless_window(bool p_borderless) { - if (current_videomode.borderless_window == p_borderless) + if (get_borderless_window() == p_borderless) return; if (!p_borderless && layered_window) @@ -1529,7 +1529,24 @@ void OS_X11::set_borderless_window(bool p_borderless) { } bool OS_X11::get_borderless_window() { - return current_videomode.borderless_window; + + bool borderless = current_videomode.borderless_window; + Atom prop = XInternAtom(x11_display, "_MOTIF_WM_HINTS", True); + if (prop != None) { + + Atom type; + int format; + unsigned long len; + unsigned long remaining; + unsigned char *data = NULL; + if (XGetWindowProperty(x11_display, x11_window, prop, 0, sizeof(Hints), False, AnyPropertyType, &type, &format, &len, &remaining, &data) == Success) { + if (data && (format == 32) && (len >= 5)) { + borderless = !((Hints *)data)->decorations; + } + XFree(data); + } + } + return borderless; } void OS_X11::request_attention() { From 19f132b3687ef2df9c56cc8781eb287519725616 Mon Sep 17 00:00:00 2001 From: Leonardo Giovanni Scur Date: Fri, 14 Jun 2019 17:37:21 -0300 Subject: [PATCH 087/113] Fix pointer position in hidpi-corrected resolutions on web (cherry picked from commit 31cc1bdb58708101022c43aba4130f7a7366b337) --- misc/dist/html/full-size.html | 9 +++++++-- platform/javascript/os_javascript.cpp | 22 ++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/misc/dist/html/full-size.html b/misc/dist/html/full-size.html index adf3a2cf7d4..12b9af21e5b 100644 --- a/misc/dist/html/full-size.html +++ b/misc/dist/html/full-size.html @@ -162,8 +162,13 @@ $GODOT_HEAD_INCLUDE requestAnimationFrame(animate); function adjustCanvasDimensions() { - canvas.width = innerWidth; - canvas.height = innerHeight; + var scale = window.devicePixelRatio || 1; + var width = window.innerWidth; + var height = window.innerHeight; + canvas.width = width * scale; + canvas.height = height * scale; + canvas.style.width = width + "px"; + canvas.style.height = height + "px"; } animationCallbacks.push(adjustCanvasDimensions); adjustCanvasDimensions(); diff --git a/platform/javascript/os_javascript.cpp b/platform/javascript/os_javascript.cpp index 34781ce365e..3256c9b4c3a 100644 --- a/platform/javascript/os_javascript.cpp +++ b/platform/javascript/os_javascript.cpp @@ -70,6 +70,20 @@ static bool is_canvas_focused() { /* clang-format on */ } +static Point2 correct_canvas_position(int x, int y) { + int canvas_width; + int canvas_height; + emscripten_get_canvas_element_size(NULL, &canvas_width, &canvas_height); + + double element_width; + double element_height; + emscripten_get_element_css_size(NULL, &element_width, &element_height); + + x = (int)(canvas_width / element_width * x); + y = (int)(canvas_height / element_height * y); + return Point2(x, y); +} + static bool cursor_inside_canvas = true; EM_BOOL OS_JavaScript::fullscreen_change_callback(int p_event_type, const EmscriptenFullscreenChangeEvent *p_event, void *p_user_data) { @@ -285,7 +299,7 @@ EM_BOOL OS_JavaScript::mouse_button_callback(int p_event_type, const EmscriptenM Ref ev; ev.instance(); ev->set_pressed(p_event_type == EMSCRIPTEN_EVENT_MOUSEDOWN); - ev->set_position(Point2(p_event->canvasX, p_event->canvasY)); + ev->set_position(correct_canvas_position(p_event->canvasX, p_event->canvasY)); ev->set_global_position(ev->get_position()); dom2godot_mod(p_event, ev); switch (p_event->button) { @@ -349,7 +363,7 @@ EM_BOOL OS_JavaScript::mousemove_callback(int p_event_type, const EmscriptenMous OS_JavaScript *os = get_singleton(); int input_mask = os->input->get_mouse_button_mask(); - Point2 pos = Point2(p_event->canvasX, p_event->canvasY); + Point2 pos = correct_canvas_position(p_event->canvasX, p_event->canvasY); // For motion outside the canvas, only read mouse movement if dragging // started inside the canvas; imitating desktop app behaviour. if (!cursor_inside_canvas && !input_mask) @@ -666,7 +680,7 @@ EM_BOOL OS_JavaScript::touch_press_callback(int p_event_type, const EmscriptenTo if (!touch.isChanged) continue; ev->set_index(touch.identifier); - ev->set_position(Point2(touch.canvasX, touch.canvasY)); + ev->set_position(correct_canvas_position(touch.canvasX, touch.canvasY)); os->touches[i] = ev->get_position(); ev->set_pressed(p_event_type == EMSCRIPTEN_EVENT_TOUCHSTART); @@ -691,7 +705,7 @@ EM_BOOL OS_JavaScript::touchmove_callback(int p_event_type, const EmscriptenTouc if (!touch.isChanged) continue; ev->set_index(touch.identifier); - ev->set_position(Point2(touch.canvasX, touch.canvasY)); + ev->set_position(correct_canvas_position(touch.canvasX, touch.canvasY)); Point2 &prev = os->touches[i]; ev->set_relative(ev->get_position() - prev); prev = ev->get_position(); From 7592b3b9ee54130422fc266107da21333d1cdf9a Mon Sep 17 00:00:00 2001 From: Hein-Pieter van Braam-Stewart Date: Thu, 6 Jun 2019 13:46:54 +0000 Subject: [PATCH 088/113] Fix a random SSL crash I saw once. I don't know why this happened, I was debugging another issue. This should take care of it though. (cherry picked from commit 64626f682e41168a2973c5f23f35f5518c211864) --- core/io/http_client.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/io/http_client.cpp b/core/io/http_client.cpp index e5c6d2a4f29..878129e346c 100644 --- a/core/io/http_client.cpp +++ b/core/io/http_client.cpp @@ -346,6 +346,12 @@ Error HTTPClient::poll() { } else { // We are already handshaking, which means we can use your already active SSL connection ssl = static_cast >(connection); + if (ssl.is_null()) { + close(); + status = STATUS_SSL_HANDSHAKE_ERROR; + return ERR_CANT_CONNECT; + } + ssl->poll(); // Try to finish the handshake } From 8148732bd141f4435fac561446079b5eaa9c9343 Mon Sep 17 00:00:00 2001 From: Paulb23 Date: Fri, 14 Jun 2019 21:48:16 +0100 Subject: [PATCH 089/113] Fix TextEdit cursor.column having a negative value (cherry picked from commit 1d55207fd2d80d8dec0d246bda69117fe211b74c) --- scene/gui/text_edit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scene/gui/text_edit.cpp b/scene/gui/text_edit.cpp index 986f2e5d4cd..79508436550 100644 --- a/scene/gui/text_edit.cpp +++ b/scene/gui/text_edit.cpp @@ -3891,7 +3891,7 @@ void TextEdit::cursor_set_line(int p_row, bool p_adjust_viewport, bool p_can_be_ cursor.line = p_row; int n_col = get_char_pos_for_line(cursor.last_fit_x, p_row, p_wrap_index); - if (is_wrap_enabled() && p_wrap_index < times_line_wraps(p_row)) { + if (n_col != 0 && is_wrap_enabled() && p_wrap_index < times_line_wraps(p_row)) { Vector rows = get_wrap_rows_text(p_row); int row_end_col = 0; for (int i = 0; i < p_wrap_index + 1; i++) { From cfd9121fa6f632fc7fd3074030e2829b3f3dac53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Sat, 15 Jun 2019 15:16:11 +0200 Subject: [PATCH 090/113] glTF: Fix import of animations with INTERPOLATION_LINEAR Bug found thanks to GCC 8's -Wduplicated-branches. Slight refactor for readability. (cherry picked from commit b486f5dde03e71ddd53571ba49c992f4c897a88a) --- editor/import/editor_scene_importer_gltf.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/editor/import/editor_scene_importer_gltf.cpp b/editor/import/editor_scene_importer_gltf.cpp index e6e29df1338..e40e87668f4 100644 --- a/editor/import/editor_scene_importer_gltf.cpp +++ b/editor/import/editor_scene_importer_gltf.cpp @@ -2087,22 +2087,23 @@ void EditorSceneImporterGLTF::_import_animation(GLTFState &state, AnimationPlaye animation->add_track(Animation::TYPE_VALUE); animation->track_set_path(track_idx, node_path); - if (track.weight_tracks[i].interpolation <= GLTFAnimation::INTERP_STEP) { - animation->track_set_interpolation_type(track_idx, track.weight_tracks[i].interpolation == GLTFAnimation::INTERP_STEP ? Animation::INTERPOLATION_NEAREST : Animation::INTERPOLATION_NEAREST); + // Only LINEAR and STEP (NEAREST) can be supported out of the box by Godot's Animation, + // the other modes have to be baked. + GLTFAnimation::Interpolation gltf_interp = track.weight_tracks[i].interpolation; + if (gltf_interp == GLTFAnimation::INTERP_LINEAR || gltf_interp == GLTFAnimation::INTERP_STEP) { + animation->track_set_interpolation_type(track_idx, gltf_interp == GLTFAnimation::INTERP_STEP ? Animation::INTERPOLATION_NEAREST : Animation::INTERPOLATION_LINEAR); for (int j = 0; j < track.weight_tracks[i].times.size(); j++) { float t = track.weight_tracks[i].times[j]; float w = track.weight_tracks[i].values[j]; animation->track_insert_key(track_idx, t, w); } } else { - //must bake, apologies. + // CATMULLROMSPLINE or CUBIC_SPLINE have to be baked, apologies. float increment = 1.0 / float(bake_fps); float time = 0.0; - bool last = false; while (true) { - - _interpolate_track(track.weight_tracks[i].times, track.weight_tracks[i].values, time, track.weight_tracks[i].interpolation); + _interpolate_track(track.weight_tracks[i].times, track.weight_tracks[i].values, time, gltf_interp); if (last) { break; } From 229892c40bbd91ab365d278701c181367e37239f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 18 Jun 2019 11:27:43 +0200 Subject: [PATCH 091/113] GDScript: Improve error on Object to Object invalid argument calls Fixes #27804. (cherry picked from commit 6750e1b3cdafbb3acb2767ec049483527a11d862) --- modules/gdscript/gdscript_function.cpp | 62 +++++++++++++++----------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/modules/gdscript/gdscript_function.cpp b/modules/gdscript/gdscript_function.cpp index cff9ba55b86..bae5eca2189 100644 --- a/modules/gdscript/gdscript_function.cpp +++ b/modules/gdscript/gdscript_function.cpp @@ -133,35 +133,13 @@ Variant *GDScriptFunction::_get_variant(int p_address, GDScriptInstance *p_insta return NULL; } -String GDScriptFunction::_get_call_error(const Variant::CallError &p_err, const String &p_where, const Variant **argptrs) const { - - String err_text; - - if (p_err.error == Variant::CallError::CALL_ERROR_INVALID_ARGUMENT) { - int errorarg = p_err.argument; - err_text = "Invalid type in " + p_where + ". Cannot convert argument " + itos(errorarg + 1) + " from " + Variant::get_type_name(argptrs[errorarg]->get_type()) + " to " + Variant::get_type_name(p_err.expected) + "."; - } else if (p_err.error == Variant::CallError::CALL_ERROR_TOO_MANY_ARGUMENTS) { - err_text = "Invalid call to " + p_where + ". Expected " + itos(p_err.argument) + " arguments."; - } else if (p_err.error == Variant::CallError::CALL_ERROR_TOO_FEW_ARGUMENTS) { - err_text = "Invalid call to " + p_where + ". Expected " + itos(p_err.argument) + " arguments."; - } else if (p_err.error == Variant::CallError::CALL_ERROR_INVALID_METHOD) { - err_text = "Invalid call. Nonexistent " + p_where + "."; - } else if (p_err.error == Variant::CallError::CALL_ERROR_INSTANCE_IS_NULL) { - err_text = "Attempt to call " + p_where + " on a null instance."; - } else { - err_text = "Bug, call error: #" + itos(p_err.error); - } - - return err_text; -} - #ifdef DEBUG_ENABLED -static String _get_var_type(const Variant *p_type) { +static String _get_var_type(const Variant *p_var) { String basestr; - if (p_type->get_type() == Variant::OBJECT) { - Object *bobj = *p_type; + if (p_var->get_type() == Variant::OBJECT) { + Object *bobj = *p_var; if (!bobj) { basestr = "null instance"; } else { @@ -176,12 +154,42 @@ static String _get_var_type(const Variant *p_type) { } } else { - basestr = Variant::get_type_name(p_type->get_type()); + basestr = Variant::get_type_name(p_var->get_type()); } return basestr; } -#endif +#endif // DEBUG_ENABLED + +String GDScriptFunction::_get_call_error(const Variant::CallError &p_err, const String &p_where, const Variant **argptrs) const { + + String err_text; + + if (p_err.error == Variant::CallError::CALL_ERROR_INVALID_ARGUMENT) { + int errorarg = p_err.argument; + // Handle the Object to Object case separately as we don't have further class details. +#ifdef DEBUG_ENABLED + if (p_err.expected == Variant::OBJECT && argptrs[errorarg]->get_type() == p_err.expected) { + err_text = "Invalid type in " + p_where + ". The Object-derived class of argument " + itos(errorarg + 1) + " (" + _get_var_type(argptrs[errorarg]) + ") is not a subclass of the expected argument class."; + } else +#endif // DEBUG_ENABLED + { + err_text = "Invalid type in " + p_where + ". Cannot convert argument " + itos(errorarg + 1) + " from " + Variant::get_type_name(argptrs[errorarg]->get_type()) + " to " + Variant::get_type_name(p_err.expected) + "."; + } + } else if (p_err.error == Variant::CallError::CALL_ERROR_TOO_MANY_ARGUMENTS) { + err_text = "Invalid call to " + p_where + ". Expected " + itos(p_err.argument) + " arguments."; + } else if (p_err.error == Variant::CallError::CALL_ERROR_TOO_FEW_ARGUMENTS) { + err_text = "Invalid call to " + p_where + ". Expected " + itos(p_err.argument) + " arguments."; + } else if (p_err.error == Variant::CallError::CALL_ERROR_INVALID_METHOD) { + err_text = "Invalid call. Nonexistent " + p_where + "."; + } else if (p_err.error == Variant::CallError::CALL_ERROR_INSTANCE_IS_NULL) { + err_text = "Attempt to call " + p_where + " on a null instance."; + } else { + err_text = "Bug, call error: #" + itos(p_err.error); + } + + return err_text; +} #if defined(__GNUC__) #define OPCODES_TABLE \ From e7e7d17f6b190d32e22d396cc58c89f7794a3430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Sun, 23 Jun 2019 19:52:49 +0200 Subject: [PATCH 092/113] Display invalid value keys in AnimationTrackEdit Godot 2.1 and 3.0 had this feature but it was lost in the rewrite of the animation editor in 3.1. Drop unused KeyValid icon, since all valid keys now have a custom type icon. (cherry picked from commit b3d772ed837c57d590c064f9de4e2ea89eb55001) --- editor/animation_track_editor.cpp | 81 +++++++++++++++++-------------- editor/animation_track_editor.h | 14 +++--- editor/icons/icon_key_valid.svg | 5 -- 3 files changed, 51 insertions(+), 49 deletions(-) delete mode 100644 editor/icons/icon_key_valid.svg diff --git a/editor/animation_track_editor.cpp b/editor/animation_track_editor.cpp index d9bff1e0e68..536b66f198b 100644 --- a/editor/animation_track_editor.cpp +++ b/editor/animation_track_editor.cpp @@ -1205,7 +1205,9 @@ AnimationTimelineEdit::AnimationTimelineEdit() { //////////////////////////////////// void AnimationTrackEdit::_notification(int p_what) { + if (p_what == NOTIFICATION_DRAW) { + if (animation.is_null()) return; ERR_FAIL_INDEX(track, animation->get_track_count()); @@ -1241,20 +1243,15 @@ void AnimationTrackEdit::_notification(int p_what) { int ofs = in_group ? check->get_width() : 0; //not the best reference for margin but.. check_rect = Rect2(Point2(ofs, int(get_size().height - check->get_height()) / 2), check->get_size()); - draw_texture(check, check_rect.position); - ofs += check->get_width() + hsep; Ref type_icon = type_icons[animation->track_get_type(track)]; - draw_texture(type_icon, Point2(ofs, int(get_size().height - type_icon->get_height()) / 2)); ofs += type_icon->get_width() + hsep; NodePath path = animation->track_get_path(track); - Node *node = NULL; - if (root && root->has_node(path)) { node = root->get_node(path); } @@ -1319,12 +1316,11 @@ void AnimationTrackEdit::_notification(int p_what) { draw_line(Point2(limit, 0), Point2(limit, get_size().height), linecolor, Math::round(EDSCALE)); } - // KEYFAMES // + // KEYFRAMES // draw_bg(limit, get_size().width - timeline->get_buttons_width()); { - float scale = timeline->get_zoom_scale(); int limit_end = get_size().width - timeline->get_buttons_width(); @@ -1353,6 +1349,7 @@ void AnimationTrackEdit::_notification(int p_what) { draw_fg(limit, get_size().width - timeline->get_buttons_width()); // BUTTONS // + { Ref wrap_icon[2] = { @@ -1577,7 +1574,18 @@ void AnimationTrackEdit::draw_key(int p_index, float p_pixels_sec, int p_x, bool if (p_x < p_clip_left || p_x > p_clip_right) return; - Vector2 ofs(p_x - type_icon->get_width() / 2, int(get_size().height - type_icon->get_height()) / 2); + Ref icon_to_draw = p_selected ? selected_icon : type_icon; + + // Override type icon for invalid value keys, unless selected. + if (!p_selected && animation->track_get_type(track) == Animation::TYPE_VALUE) { + const Variant &v = animation->track_get_key_value(track, p_index); + Variant::Type valid_type = Variant::NIL; + if (!_is_value_key_valid(v, valid_type)) { + icon_to_draw = get_icon("KeyInvalid", "EditorIcons"); + } + } + + Vector2 ofs(p_x - icon_to_draw->get_width() / 2, int(get_size().height - icon_to_draw->get_height()) / 2); if (animation->track_get_type(track) == Animation::TYPE_METHOD) { Ref font = get_font("font", "Label"); @@ -1601,16 +1609,13 @@ void AnimationTrackEdit::draw_key(int p_index, float p_pixels_sec, int p_x, bool } text += ")"; - int limit = MAX(0, p_clip_right - p_x - type_icon->get_width()); + int limit = MAX(0, p_clip_right - p_x - icon_to_draw->get_width()); if (limit > 0) { - draw_string(font, Vector2(p_x + type_icon->get_width(), int(get_size().height - font->get_height()) / 2 + font->get_ascent()), text, color, limit); + draw_string(font, Vector2(p_x + icon_to_draw->get_width(), int(get_size().height - font->get_height()) / 2 + font->get_ascent()), text, color, limit); } } - if (p_selected) { - draw_texture(selected_icon, ofs); - } else { - draw_texture(type_icon, ofs); - } + + draw_texture(icon_to_draw, ofs); } //helper @@ -1775,6 +1780,27 @@ void AnimationTrackEdit::_path_entered(const String &p_text) { undo_redo->commit_action(); } +bool AnimationTrackEdit::_is_value_key_valid(const Variant &p_key_value, Variant::Type &r_valid_type) const { + + RES res; + Vector leftover_path; + Node *node = root->get_node_and_resource(animation->track_get_path(track), res, leftover_path); + + Object *obj = NULL; + if (res.is_valid()) { + obj = res.ptr(); + } else if (node) { + obj = node; + } + + bool prop_exists = false; + if (obj) { + r_valid_type = obj->get_static_property_type_indexed(leftover_path, &prop_exists); + } + + return (!prop_exists || Variant::can_convert(p_key_value.get_type(), r_valid_type)); +} + String AnimationTrackEdit::get_tooltip(const Point2 &p_pos) const { if (check_rect.has_point(p_pos)) { @@ -1845,29 +1871,10 @@ String AnimationTrackEdit::get_tooltip(const Point2 &p_pos) const { } break; case Animation::TYPE_VALUE: { - Variant v = animation->track_get_key_value(track, key_idx); - //text+="value: "+String(v)+"\n"; - - bool prop_exists = false; - Variant::Type valid_type = Variant::NIL; - Object *obj = NULL; - - RES res; - Vector leftover_path; - Node *node = root->get_node_and_resource(animation->track_get_path(track), res, leftover_path); - - if (res.is_valid()) { - obj = res.ptr(); - } else if (node) { - obj = node; - } - - if (obj) { - valid_type = obj->get_static_property_type_indexed(leftover_path, &prop_exists); - } - + const Variant &v = animation->track_get_key_value(track, key_idx); text += "Type: " + Variant::get_type_name(v.get_type()) + "\n"; - if (prop_exists && !Variant::can_convert(v.get_type(), valid_type)) { + Variant::Type valid_type = Variant::NIL; + if (!_is_value_key_valid(v, valid_type)) { text += "Value: " + String(v) + " (Invalid, expected type: " + Variant::get_type_name(valid_type) + ")\n"; } else { text += "Value: " + String(v) + "\n"; diff --git a/editor/animation_track_editor.h b/editor/animation_track_editor.h index 5ac5999b681..5603b41013c 100644 --- a/editor/animation_track_editor.h +++ b/editor/animation_track_editor.h @@ -31,6 +31,11 @@ #ifndef ANIMATION_TRACK_EDITOR_H #define ANIMATION_TRACK_EDITOR_H +#include "editor/editor_data.h" +#include "editor/editor_spin_slider.h" +#include "editor/property_editor.h" +#include "editor/property_selector.h" +#include "scene/animation/animation_cache.h" #include "scene/gui/control.h" #include "scene/gui/file_dialog.h" #include "scene/gui/menu_button.h" @@ -40,12 +45,6 @@ #include "scene/gui/tab_container.h" #include "scene/gui/texture_rect.h" #include "scene/gui/tool_button.h" - -#include "editor/property_selector.h" -#include "editor_data.h" -#include "editor_spin_slider.h" -#include "property_editor.h" -#include "scene/animation/animation_cache.h" #include "scene/resources/animation.h" #include "scene_tree_editor.h" @@ -175,8 +174,9 @@ class AnimationTrackEdit : public Control { void _path_entered(const String &p_text); void _play_position_draw(); - mutable int dropping_at; + bool _is_value_key_valid(const Variant &p_key_value, Variant::Type &r_valid_type) const; + mutable int dropping_at; float insert_at_pos; bool moving_selection_attempt; int select_single_attempt; diff --git a/editor/icons/icon_key_valid.svg b/editor/icons/icon_key_valid.svg deleted file mode 100644 index 4a3fab47548..00000000000 --- a/editor/icons/icon_key_valid.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - From 7a9eb587fd929a903c999f449ccb6314ec235ab5 Mon Sep 17 00:00:00 2001 From: Gwyneth Lowe Date: Mon, 24 Jun 2019 00:17:27 -0500 Subject: [PATCH 093/113] Correct typo that broke custom selected font color Change several font_selected_color to font_color_selected; the actual name of the override (cherry picked from commit 7142064110936593d15409dcf35c566638f87d20) --- editor/plugins/script_text_editor.cpp | 2 +- editor/plugins/shader_editor_plugin.cpp | 2 +- editor/plugins/text_editor.cpp | 2 +- scene/gui/text_edit.cpp | 12 ++++++------ scene/gui/text_edit.h | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/editor/plugins/script_text_editor.cpp b/editor/plugins/script_text_editor.cpp index f7a3ded4515..d547de75493 100644 --- a/editor/plugins/script_text_editor.cpp +++ b/editor/plugins/script_text_editor.cpp @@ -165,7 +165,7 @@ void ScriptTextEditor::_load_theme_settings() { text_edit->add_color_override("safe_line_number_color", safe_line_number_color); text_edit->add_color_override("caret_color", caret_color); text_edit->add_color_override("caret_background_color", caret_background_color); - text_edit->add_color_override("font_selected_color", text_selected_color); + text_edit->add_color_override("font_color_selected", text_selected_color); text_edit->add_color_override("selection_color", selection_color); text_edit->add_color_override("brace_mismatch_color", brace_mismatch_color); text_edit->add_color_override("current_line_color", current_line_color); diff --git a/editor/plugins/shader_editor_plugin.cpp b/editor/plugins/shader_editor_plugin.cpp index 2b6ceac8e2a..428dd28c097 100644 --- a/editor/plugins/shader_editor_plugin.cpp +++ b/editor/plugins/shader_editor_plugin.cpp @@ -102,7 +102,7 @@ void ShaderTextEditor::_load_theme_settings() { get_text_edit()->add_color_override("line_number_color", line_number_color); get_text_edit()->add_color_override("caret_color", caret_color); get_text_edit()->add_color_override("caret_background_color", caret_background_color); - get_text_edit()->add_color_override("font_selected_color", text_selected_color); + get_text_edit()->add_color_override("font_color_selected", text_selected_color); get_text_edit()->add_color_override("selection_color", selection_color); get_text_edit()->add_color_override("brace_mismatch_color", brace_mismatch_color); get_text_edit()->add_color_override("current_line_color", current_line_color); diff --git a/editor/plugins/text_editor.cpp b/editor/plugins/text_editor.cpp index fe32c97a645..33bb9ebeebb 100644 --- a/editor/plugins/text_editor.cpp +++ b/editor/plugins/text_editor.cpp @@ -115,7 +115,7 @@ void TextEditor::_load_theme_settings() { text_edit->add_color_override("line_number_color", line_number_color); text_edit->add_color_override("caret_color", caret_color); text_edit->add_color_override("caret_background_color", caret_background_color); - text_edit->add_color_override("font_selected_color", text_selected_color); + text_edit->add_color_override("font_color_selected", text_selected_color); text_edit->add_color_override("selection_color", selection_color); text_edit->add_color_override("brace_mismatch_color", brace_mismatch_color); text_edit->add_color_override("current_line_color", current_line_color); diff --git a/scene/gui/text_edit.cpp b/scene/gui/text_edit.cpp index 79508436550..6e5a078def7 100644 --- a/scene/gui/text_edit.cpp +++ b/scene/gui/text_edit.cpp @@ -1095,7 +1095,7 @@ void TextEdit::_notification(int p_what) { if (brace_open_mismatch) color = cache.brace_mismatch_color; - drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), '_', str[j + 1], in_selection && override_selected_font_color ? cache.font_selected_color : color); + drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), '_', str[j + 1], in_selection && override_selected_font_color ? cache.font_color_selected : color); } if ((brace_close_match_line == line && brace_close_match_column == last_wrap_column + j) || @@ -1103,7 +1103,7 @@ void TextEdit::_notification(int p_what) { if (brace_close_mismatch) color = cache.brace_mismatch_color; - drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), '_', str[j + 1], in_selection && override_selected_font_color ? cache.font_selected_color : color); + drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), '_', str[j + 1], in_selection && override_selected_font_color ? cache.font_color_selected : color); } } @@ -1175,18 +1175,18 @@ void TextEdit::_notification(int p_what) { if (str[j] >= 32) { int yofs = ofs_y + (get_row_height() - cache.font->get_height()) / 2; - int w = drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), str[j], str[j + 1], in_selection && override_selected_font_color ? cache.font_selected_color : color); + int w = drawer.draw_char(ci, Point2i(char_ofs + char_margin + ofs_x, yofs + ascent), str[j], str[j + 1], in_selection && override_selected_font_color ? cache.font_color_selected : color); if (underlined) { float line_width = 1.0; #ifdef TOOLS_ENABLED line_width *= EDSCALE; #endif - draw_rect(Rect2(char_ofs + char_margin + ofs_x, yofs + ascent + 2, w, line_width), in_selection && override_selected_font_color ? cache.font_selected_color : color); + draw_rect(Rect2(char_ofs + char_margin + ofs_x, yofs + ascent + 2, w, line_width), in_selection && override_selected_font_color ? cache.font_color_selected : color); } } else if (draw_tabs && str[j] == '\t') { int yofs = (get_row_height() - cache.tab_icon->get_height()) / 2; - cache.tab_icon->draw(ci, Point2(char_ofs + char_margin + ofs_x, ofs_y + yofs), in_selection && override_selected_font_color ? cache.font_selected_color : color); + cache.tab_icon->draw(ci, Point2(char_ofs + char_margin + ofs_x, ofs_y + yofs), in_selection && override_selected_font_color ? cache.font_color_selected : color); } char_ofs += char_w; @@ -4333,7 +4333,7 @@ void TextEdit::_update_caches() { cache.line_number_color = get_color("line_number_color"); cache.safe_line_number_color = get_color("safe_line_number_color"); cache.font_color = get_color("font_color"); - cache.font_selected_color = get_color("font_selected_color"); + cache.font_color_selected = get_color("font_color_selected"); cache.keyword_color = get_color("keyword_color"); cache.function_color = get_color("function_color"); cache.member_variable_color = get_color("member_variable_color"); diff --git a/scene/gui/text_edit.h b/scene/gui/text_edit.h index 33f0a3f45dc..1f9f3a358f3 100644 --- a/scene/gui/text_edit.h +++ b/scene/gui/text_edit.h @@ -170,7 +170,7 @@ private: Color line_number_color; Color safe_line_number_color; Color font_color; - Color font_selected_color; + Color font_color_selected; Color keyword_color; Color number_color; Color function_color; From aa08c7c41849738a2440dea12a3330ba278caec7 Mon Sep 17 00:00:00 2001 From: Marcus Brummer Date: Mon, 24 Jun 2019 16:00:54 +0200 Subject: [PATCH 094/113] Check if autoload nodes are != NULL before deleting them. This fixes #27854 (cherry picked from commit 0716c59f14383343c1761bc2e0327ebdb2d35d2d) --- editor/editor_autoload_settings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editor/editor_autoload_settings.cpp b/editor/editor_autoload_settings.cpp index 2f2b0d2cba7..2fb4e3fe1bc 100644 --- a/editor/editor_autoload_settings.cpp +++ b/editor/editor_autoload_settings.cpp @@ -772,7 +772,7 @@ EditorAutoloadSettings::EditorAutoloadSettings() { } } - if (!info.is_singleton && !info.in_editor) { + if (!info.is_singleton && !info.in_editor && info.node != NULL) { memdelete(info.node); info.node = NULL; } From 0fc4c3a3082763fba9e53a2363febf6f55aa7b1f Mon Sep 17 00:00:00 2001 From: Tomasz Chabora Date: Mon, 24 Jun 2019 21:15:26 +0200 Subject: [PATCH 095/113] Allow to remove 2D editor limits (cherry picked from commit c81525ec055b9941305d300f073046d7bc79b400) --- editor/editor_settings.cpp | 1 + editor/plugins/canvas_item_editor_plugin.cpp | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/editor/editor_settings.cpp b/editor/editor_settings.cpp index f477134f5a1..1aa191e6ae9 100644 --- a/editor/editor_settings.cpp +++ b/editor/editor_settings.cpp @@ -557,6 +557,7 @@ void EditorSettings::_load_defaults(Ref p_extra_config) { _initial_set("editors/2d/bone_outline_size", 2); _initial_set("editors/2d/keep_margins_when_changing_anchors", false); _initial_set("editors/2d/viewport_border_color", Color(0.4, 0.4, 1.0, 0.4)); + _initial_set("editors/2d/constrain_editor_view", true); _initial_set("editors/2d/warped_mouse_panning", true); _initial_set("editors/2d/simple_spacebar_panning", false); _initial_set("editors/2d/scroll_to_pan", false); diff --git a/editor/plugins/canvas_item_editor_plugin.cpp b/editor/plugins/canvas_item_editor_plugin.cpp index 30c78722490..30d9974d84e 100644 --- a/editor/plugins/canvas_item_editor_plugin.cpp +++ b/editor/plugins/canvas_item_editor_plugin.cpp @@ -3547,18 +3547,19 @@ void CanvasItemEditor::_update_scrollbars() { // Constraints the view offset and updates the scrollbars Point2 begin = canvas_item_rect.position; Point2 end = canvas_item_rect.position + canvas_item_rect.size - local_rect.size / zoom; + bool constrain_editor_view = bool(EditorSettings::get_singleton()->get("editors/2d/constrain_editor_view")); if (canvas_item_rect.size.height <= (local_rect.size.y / zoom)) { - if (ABS(begin.y - previous_update_view_offset.y) < ABS(begin.y - view_offset.y)) { + if (constrain_editor_view && ABS(begin.y - previous_update_view_offset.y) < ABS(begin.y - view_offset.y)) { view_offset.y = previous_update_view_offset.y; } v_scroll->hide(); } else { - if (view_offset.y > end.y && view_offset.y > previous_update_view_offset.y) { + if (constrain_editor_view && view_offset.y > end.y && view_offset.y > previous_update_view_offset.y) { view_offset.y = MAX(end.y, previous_update_view_offset.y); } - if (view_offset.y < begin.y && view_offset.y < previous_update_view_offset.y) { + if (constrain_editor_view && view_offset.y < begin.y && view_offset.y < previous_update_view_offset.y) { view_offset.y = MIN(begin.y, previous_update_view_offset.y); } @@ -3569,16 +3570,16 @@ void CanvasItemEditor::_update_scrollbars() { } if (canvas_item_rect.size.width <= (local_rect.size.x / zoom)) { - if (ABS(begin.x - previous_update_view_offset.x) < ABS(begin.x - view_offset.x)) { + if (constrain_editor_view && ABS(begin.x - previous_update_view_offset.x) < ABS(begin.x - view_offset.x)) { view_offset.x = previous_update_view_offset.x; } h_scroll->hide(); } else { - if (view_offset.x > end.x && view_offset.x > previous_update_view_offset.x) { + if (constrain_editor_view && view_offset.x > end.x && view_offset.x > previous_update_view_offset.x) { view_offset.x = MAX(end.x, previous_update_view_offset.x); } - if (view_offset.x < begin.x && view_offset.x < previous_update_view_offset.x) { + if (constrain_editor_view && view_offset.x < begin.x && view_offset.x < previous_update_view_offset.x) { view_offset.x = MIN(begin.x, previous_update_view_offset.x); } From 1c8984c4332e97e58bce1b67845587597594e96b Mon Sep 17 00:00:00 2001 From: santouits Date: Thu, 27 Jun 2019 03:25:36 +0300 Subject: [PATCH 096/113] Free server id pools directly. When closing the game, we flush the command queue but after we are pushing the freeing calls of the id pool to the command queue and they are never being run. Now we free them directly. (cherry picked from commit 7aa4622222d3cc88b2d6c2e5552bc372272c3123) --- servers/server_wrap_mt_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servers/server_wrap_mt_common.h b/servers/server_wrap_mt_common.h index f0ab2563749..a6f595d7760 100644 --- a/servers/server_wrap_mt_common.h +++ b/servers/server_wrap_mt_common.h @@ -50,7 +50,7 @@ } \ void m_type##_free_cached_ids() { \ while (m_type##_id_pool.size()) { \ - free(m_type##_id_pool.front()->get()); \ + server_name->free(m_type##_id_pool.front()->get()); \ m_type##_id_pool.pop_front(); \ } \ } \ From 0f0016bb75b9cc639602d645071d02e1b9418bde Mon Sep 17 00:00:00 2001 From: zwostein Date: Mon, 1 Jul 2019 22:03:02 +0200 Subject: [PATCH 097/113] Explicitly set a channel map when initialising the Pulseaudio stream. (cherry picked from commit 9fa2e07588eb91b72ee4e6abdb30458673c88498) --- drivers/pulseaudio/audio_driver_pulseaudio.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/pulseaudio/audio_driver_pulseaudio.cpp b/drivers/pulseaudio/audio_driver_pulseaudio.cpp index aec3d27d692..a61fa449f11 100644 --- a/drivers/pulseaudio/audio_driver_pulseaudio.cpp +++ b/drivers/pulseaudio/audio_driver_pulseaudio.cpp @@ -191,6 +191,14 @@ Error AudioDriverPulseAudio::init_device() { spec.format = PA_SAMPLE_S16LE; spec.channels = pa_map.channels; spec.rate = mix_rate; + pa_map.map[0] = PA_CHANNEL_POSITION_FRONT_LEFT; + pa_map.map[1] = PA_CHANNEL_POSITION_FRONT_RIGHT; + pa_map.map[2] = PA_CHANNEL_POSITION_FRONT_CENTER; + pa_map.map[3] = PA_CHANNEL_POSITION_LFE; + pa_map.map[4] = PA_CHANNEL_POSITION_REAR_LEFT; + pa_map.map[5] = PA_CHANNEL_POSITION_REAR_RIGHT; + pa_map.map[6] = PA_CHANNEL_POSITION_SIDE_LEFT; + pa_map.map[7] = PA_CHANNEL_POSITION_SIDE_RIGHT; pa_str = pa_stream_new(pa_ctx, "Sound", &spec, &pa_map); if (pa_str == NULL) { From e17a57d88a5a9594fb1b1b4285daa851bdee0778 Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Mon, 8 Jul 2019 12:35:52 +0300 Subject: [PATCH 098/113] Fix some issue with TileMap's and other nodes' boundaries Fixes #30348 Addresses a small part of #30012 (cherry picked from commit ebf2a4d5531d9a2fe86f0628352f438e6db7dd01) --- scene/2d/path_2d.cpp | 2 +- scene/2d/polygon_2d.cpp | 2 +- scene/2d/tile_map.cpp | 14 +++++++++----- scene/2d/tile_map.h | 7 +++++++ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scene/2d/path_2d.cpp b/scene/2d/path_2d.cpp index 4097006b33b..45927b28c25 100644 --- a/scene/2d/path_2d.cpp +++ b/scene/2d/path_2d.cpp @@ -58,7 +58,7 @@ Rect2 Path2D::_edit_get_rect() const { } bool Path2D::_edit_use_rect() const { - return true; + return curve.is_valid() && curve->get_point_count() != 0; } bool Path2D::_edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const { diff --git a/scene/2d/polygon_2d.cpp b/scene/2d/polygon_2d.cpp index f6f1bad5817..32a0b732c09 100644 --- a/scene/2d/polygon_2d.cpp +++ b/scene/2d/polygon_2d.cpp @@ -76,7 +76,7 @@ Rect2 Polygon2D::_edit_get_rect() const { } bool Polygon2D::_edit_use_rect() const { - return true; + return polygon.size() > 0; } bool Polygon2D::_edit_is_selected_on_click(const Point2 &p_point, double p_tolerance) const { diff --git a/scene/2d/tile_map.cpp b/scene/2d/tile_map.cpp index 3562011bc29..788d93a6a6d 100644 --- a/scene/2d/tile_map.cpp +++ b/scene/2d/tile_map.cpp @@ -760,7 +760,7 @@ void TileMap::set_cell(int p_x, int p_y, int p_tile, bool p_flip_x, bool p_flip_ if (!E && p_tile == INVALID_CELL) return; //nothing to do - PosKey qk(p_x / _get_quadrant_size(), p_y / _get_quadrant_size()); + PosKey qk = pk.to_quadrant(_get_quadrant_size()); if (p_tile == INVALID_CELL) { //erase existing tile_map.erase(pk); @@ -919,7 +919,7 @@ void TileMap::update_cell_bitmask(int p_x, int p_y) { E->get().autotile_coord_x = (int)coord.x; E->get().autotile_coord_y = (int)coord.y; - PosKey qk(p_x / _get_quadrant_size(), p_y / _get_quadrant_size()); + PosKey qk = p.to_quadrant(_get_quadrant_size()); Map::Element *Q = quadrant_map.find(qk); _make_quadrant_dirty(Q); @@ -1007,7 +1007,7 @@ void TileMap::set_cell_autotile_coord(int p_x, int p_y, const Vector2 &p_coord) c.autotile_coord_y = p_coord.y; tile_map[pk] = c; - PosKey qk(p_x / _get_quadrant_size(), p_y / _get_quadrant_size()); + PosKey qk = pk.to_quadrant(_get_quadrant_size()); Map::Element *Q = quadrant_map.find(qk); if (!Q) @@ -1034,7 +1034,7 @@ void TileMap::_recreate_quadrants() { for (Map::Element *E = tile_map.front(); E; E = E->next()) { - PosKey qk(E->key().x / _get_quadrant_size(), E->key().y / _get_quadrant_size()); + PosKey qk = PosKey(E->key().x, E->key().y).to_quadrant(_get_quadrant_size()); Map::Element *Q = quadrant_map.find(qk); if (!Q) { @@ -1174,7 +1174,11 @@ PoolVector TileMap::_get_tile_data() const { } Rect2 TileMap::_edit_get_rect() const { - const_cast(this)->update_dirty_quadrants(); + if (pending_update) { + const_cast(this)->update_dirty_quadrants(); + } else { + const_cast(this)->_recompute_rect_cache(); + } return rect_cache; } diff --git a/scene/2d/tile_map.h b/scene/2d/tile_map.h index 6a1467aa48f..e06eab25110 100644 --- a/scene/2d/tile_map.h +++ b/scene/2d/tile_map.h @@ -90,6 +90,13 @@ private: bool operator==(const PosKey &p_k) const { return (y == p_k.y && x == p_k.x); } + PosKey to_quadrant(const int &p_quadrant_size) const { + // rounding down, instead of simply rounding towards zero (truncating) + return PosKey( + x > 0 ? x / p_quadrant_size : (x - (p_quadrant_size - 1)) / p_quadrant_size, + y > 0 ? y / p_quadrant_size : (y - (p_quadrant_size - 1)) / p_quadrant_size); + } + PosKey(int16_t p_x, int16_t p_y) { x = p_x; y = p_y; From c9422b9c235b1d68dd513a2cc34592b4feb8061b Mon Sep 17 00:00:00 2001 From: Shiqing Date: Thu, 11 Jul 2019 17:40:42 +0800 Subject: [PATCH 099/113] Fix #29478 multiple modals possibly freeze (cherry picked from commit c16e764bb2ae2f038abac8b1fc2518bba3dbf3db) --- scene/main/viewport.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scene/main/viewport.cpp b/scene/main/viewport.cpp index e31939215e6..038aedd8768 100644 --- a/scene/main/viewport.cpp +++ b/scene/main/viewport.cpp @@ -2561,7 +2561,7 @@ void Viewport::_drop_mouse_focus() { List::Element *Viewport::_gui_show_modal(Control *p_control) { - gui.modal_stack.push_back(p_control); + List::Element *node = gui.modal_stack.push_back(p_control); if (gui.key_focus) p_control->_modal_set_prev_focus_owner(gui.key_focus->get_instance_id()); else @@ -2572,7 +2572,7 @@ List::Element *Viewport::_gui_show_modal(Control *p_control) { _drop_mouse_focus(); } - return gui.modal_stack.back(); + return node; } Control *Viewport::_gui_get_focus_owner() { From 20822ee87d284b84e53058f120feb07c3eec11d6 Mon Sep 17 00:00:00 2001 From: SonerSound Date: Sat, 13 Jul 2019 23:01:17 +0100 Subject: [PATCH 100/113] Fix inconsistent lighting in GLES2 Issue was possibily being caused by duplicating a light even when that light was not in the render_light_instances array. (cherry picked from commit 545bf86d39ea3c68ed38e14a5018231295bed922) --- drivers/gles2/rasterizer_scene_gles2.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gles2/rasterizer_scene_gles2.cpp b/drivers/gles2/rasterizer_scene_gles2.cpp index 5600cc0328f..2a18db0453b 100644 --- a/drivers/gles2/rasterizer_scene_gles2.cpp +++ b/drivers/gles2/rasterizer_scene_gles2.cpp @@ -1110,8 +1110,8 @@ void RasterizerSceneGLES2::_add_geometry_with_material(RasterizerStorageGLES2::G LightInstance *li = light_instance_owner.getornull(e->instance->light_instances[i]); - if (li->light_index >= render_light_instance_count) { - continue; // too many + if (li->light_index >= render_light_instance_count || render_light_instances[li->light_index] != li) { + continue; // too many or light_index did not correspond to the light instances to be rendered } if (copy) { From 794bfe8aaa53cd4d89e3448b17c7bc11a0fb16fe Mon Sep 17 00:00:00 2001 From: Bojidar Marinov Date: Tue, 16 Jul 2019 20:58:00 +0300 Subject: [PATCH 101/113] Fix invalid write in AudioStreamSample::save_to_wav Fixes #29955 (cherry picked from commit ea949ab873a6b7ede1a877a06114d7162a31deaa) --- scene/resources/audio_stream_sample.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scene/resources/audio_stream_sample.cpp b/scene/resources/audio_stream_sample.cpp index 4b3e3920134..5b61654c5df 100644 --- a/scene/resources/audio_stream_sample.cpp +++ b/scene/resources/audio_stream_sample.cpp @@ -564,7 +564,8 @@ Error AudioStreamSample::save_to_wav(const String &p_path) { file->store_32(sub_chunk_2_size); //Subchunk2Size // Add data - PoolVector::Read read_data = get_data().read(); + PoolVector data = get_data(); + PoolVector::Read read_data = data.read(); switch (format) { case AudioStreamSample::FORMAT_8_BITS: for (unsigned int i = 0; i < data_bytes; i++) { From 0e76f2cc9581d99d3c25cfe8251216e3358d4095 Mon Sep 17 00:00:00 2001 From: clayjohn Date: Mon, 22 Jul 2019 15:03:51 -0700 Subject: [PATCH 102/113] fix contact shadows appearing when shading casting is off (cherry picked from commit b1839e5e8502927373e948365b490a5dc60facb5) --- drivers/gles3/rasterizer_scene_gles3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gles3/rasterizer_scene_gles3.cpp b/drivers/gles3/rasterizer_scene_gles3.cpp index 1b8464b8c10..0f46d6f64d2 100644 --- a/drivers/gles3/rasterizer_scene_gles3.cpp +++ b/drivers/gles3/rasterizer_scene_gles3.cpp @@ -2354,7 +2354,7 @@ void RasterizerSceneGLES3::_add_geometry_with_material(RasterizerStorageGLES3::G if (p_depth_pass) { - if (has_blend_alpha || p_material->shader->spatial.uses_depth_texture || (has_base_alpha && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) || p_material->shader->spatial.depth_draw_mode == RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_NEVER || p_material->shader->spatial.no_depth_test) + if (has_blend_alpha || p_material->shader->spatial.uses_depth_texture || (has_base_alpha && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) || p_material->shader->spatial.depth_draw_mode == RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_NEVER || p_material->shader->spatial.no_depth_test || p_instance->cast_shadows == VS::SHADOW_CASTING_SETTING_OFF) return; //bye if (!p_material->shader->spatial.uses_alpha_scissor && !p_material->shader->spatial.writes_modelview_or_projection && !p_material->shader->spatial.uses_vertex && !p_material->shader->spatial.uses_discard && p_material->shader->spatial.depth_draw_mode != RasterizerStorageGLES3::Shader::Spatial::DEPTH_DRAW_ALPHA_PREPASS) { From 7ebfb73a4a9ec05f2d29dab8dae6a6b146f128a4 Mon Sep 17 00:00:00 2001 From: santouits Date: Sun, 28 Jul 2019 23:19:44 +0300 Subject: [PATCH 103/113] Hopefully fix the random crashes with threads (cherry picked from commit 13efc1bb062a64d513618b136bfd3d888adb2ec5) --- core/object.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/object.h b/core/object.h index e8a6178052c..603f2eadb6c 100644 --- a/core/object.h +++ b/core/object.h @@ -779,8 +779,13 @@ public: static int get_object_count(); _FORCE_INLINE_ static bool instance_validate(Object *p_ptr) { + rw_lock->read_lock(); - return instance_checks.has(p_ptr); + bool exists = instance_checks.has(p_ptr); + + rw_lock->read_unlock(); + + return exists; } }; From b5d8154d170c7e0820ec170e5951087fce9b9868 Mon Sep 17 00:00:00 2001 From: Yuri Roubinski Date: Fri, 2 Aug 2019 17:03:45 +0300 Subject: [PATCH 104/113] Added link to docs for shader editor (cherry picked from commit 770ce20ea3b8e4e75d4b9e74dd124726583acb1c) --- editor/plugins/shader_editor_plugin.cpp | 10 ++++++++++ editor/plugins/shader_editor_plugin.h | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/editor/plugins/shader_editor_plugin.cpp b/editor/plugins/shader_editor_plugin.cpp index 428dd28c097..8e5a3df5edd 100644 --- a/editor/plugins/shader_editor_plugin.cpp +++ b/editor/plugins/shader_editor_plugin.cpp @@ -346,6 +346,9 @@ void ShaderEditor::_menu_option(int p_option) { goto_line_dialog->popup_find_line(shader_editor->get_text_edit()); } break; + case HELP_DOCS: { + OS::get_singleton()->shell_open("https://docs.godotengine.org/en/stable/tutorials/shading/shading_reference/index.html"); + } break; } if (p_option != SEARCH_FIND && p_option != SEARCH_REPLACE && p_option != SEARCH_GOTO_LINE) { shader_editor->get_text_edit()->call_deferred("grab_focus"); @@ -577,10 +580,17 @@ ShaderEditor::ShaderEditor(EditorNode *p_node) { search_menu->get_popup()->add_shortcut(ED_GET_SHORTCUT("script_text_editor/goto_line"), SEARCH_GOTO_LINE); search_menu->get_popup()->connect("id_pressed", this, "_menu_option"); + help_menu = memnew(MenuButton); + help_menu->set_text(TTR("Help")); + help_menu->set_switch_on_hover(true); + help_menu->get_popup()->add_icon_item(p_node->get_gui_base()->get_icon("Instance", "EditorIcons"), TTR("Online Docs"), HELP_DOCS); + help_menu->get_popup()->connect("id_pressed", this, "_menu_option"); + add_child(main_container); main_container->add_child(hbc); hbc->add_child(search_menu); hbc->add_child(edit_menu); + hbc->add_child(help_menu); hbc->add_style_override("panel", p_node->get_gui_base()->get_stylebox("ScriptEditorPanel", "EditorStyles")); main_container->add_child(shader_editor); diff --git a/editor/plugins/shader_editor_plugin.h b/editor/plugins/shader_editor_plugin.h index 46c78c1d33e..c8b5956b0dd 100644 --- a/editor/plugins/shader_editor_plugin.h +++ b/editor/plugins/shader_editor_plugin.h @@ -88,12 +88,12 @@ class ShaderEditor : public PanelContainer { SEARCH_FIND_PREV, SEARCH_REPLACE, SEARCH_GOTO_LINE, - + HELP_DOCS, }; MenuButton *edit_menu; MenuButton *search_menu; - MenuButton *settings_menu; + MenuButton *help_menu; PopupMenu *context_menu; uint64_t idle; From 9bf2e54ca5c871223884946db953e226a17aee9c Mon Sep 17 00:00:00 2001 From: iwek7 Date: Fri, 2 Aug 2019 22:32:33 +0200 Subject: [PATCH 105/113] Fixes color picker in code editor - now it only changes one color (cherry picked from commit e7b07e1e58c5931585295b59a2714a1188a67fb9) --- editor/plugins/script_text_editor.cpp | 16 ++++++++++++---- editor/plugins/script_text_editor.h | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/editor/plugins/script_text_editor.cpp b/editor/plugins/script_text_editor.cpp index d547de75493..ff574b74547 100644 --- a/editor/plugins/script_text_editor.cpp +++ b/editor/plugins/script_text_editor.cpp @@ -1329,7 +1329,9 @@ void ScriptTextEditor::_text_edit_gui_input(const Ref &ev) { if (has_color) { String line = tx->get_line(row); - color_line = row; + color_position.x = row; + color_position.y = col; + int begin = 0; int end = 0; bool valid = false; @@ -1369,10 +1371,15 @@ void ScriptTextEditor::_color_changed(const Color &p_color) { new_args = String("(" + rtos(p_color.r) + ", " + rtos(p_color.g) + ", " + rtos(p_color.b) + ", " + rtos(p_color.a) + ")"); } - String line = code_editor->get_text_edit()->get_line(color_line); - String new_line = line.replace(color_args, new_args); + String line = code_editor->get_text_edit()->get_line(color_position.x); + int color_args_pos = line.find(color_args, color_position.y); + String line_with_replaced_args = line; + line_with_replaced_args.erase(color_args_pos, color_args.length()); + line_with_replaced_args = line_with_replaced_args.insert(color_args_pos, new_args); + color_args = new_args; - code_editor->get_text_edit()->set_line(color_line, new_line); + code_editor->get_text_edit()->set_line(color_position.x, line_with_replaced_args); + code_editor->get_text_edit()->update(); } void ScriptTextEditor::_make_context_menu(bool p_selection, bool p_color, bool p_foldable, bool p_open_docs, bool p_goto_definition) { @@ -1465,6 +1472,7 @@ ScriptTextEditor::ScriptTextEditor() { color_panel = memnew(PopupPanel); add_child(color_panel); color_picker = memnew(ColorPicker); + color_picker->set_deferred_mode(true); color_panel->add_child(color_picker); color_picker->connect("color_changed", this, "_color_changed"); diff --git a/editor/plugins/script_text_editor.h b/editor/plugins/script_text_editor.h index f83aadddef9..0ad4e6d3f90 100644 --- a/editor/plugins/script_text_editor.h +++ b/editor/plugins/script_text_editor.h @@ -59,7 +59,7 @@ class ScriptTextEditor : public ScriptEditorBase { PopupPanel *color_panel; ColorPicker *color_picker; - int color_line; + Vector2 color_position; String color_args; void _update_member_keywords(); From dd83296155966aadc45c1d9048468f51a83ad756 Mon Sep 17 00:00:00 2001 From: Cameron Reikes Date: Wed, 7 Aug 2019 23:18:09 -0700 Subject: [PATCH 106/113] Add particles icon to CPUParticles (cherry picked from commit 345fcb96a2035fd7d6d72632560b2f874a800a02) --- editor/icons/icon_gizmo_c_p_u_particles.svg | 85 +++++++++++++++++++++ editor/plugins/spatial_editor_plugin.cpp | 1 + editor/spatial_editor_gizmos.cpp | 28 +++++++ editor/spatial_editor_gizmos.h | 12 +++ 4 files changed, 126 insertions(+) create mode 100644 editor/icons/icon_gizmo_c_p_u_particles.svg diff --git a/editor/icons/icon_gizmo_c_p_u_particles.svg b/editor/icons/icon_gizmo_c_p_u_particles.svg new file mode 100644 index 00000000000..a6f0eaef5b8 --- /dev/null +++ b/editor/icons/icon_gizmo_c_p_u_particles.svg @@ -0,0 +1,85 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/editor/plugins/spatial_editor_plugin.cpp b/editor/plugins/spatial_editor_plugin.cpp index f4262e8235b..cfae2a08e61 100644 --- a/editor/plugins/spatial_editor_plugin.cpp +++ b/editor/plugins/spatial_editor_plugin.cpp @@ -5323,6 +5323,7 @@ void SpatialEditor::_register_all_gizmos() { add_gizmo_plugin(Ref(memnew(VehicleWheelSpatialGizmoPlugin))); add_gizmo_plugin(Ref(memnew(VisibilityNotifierGizmoPlugin))); add_gizmo_plugin(Ref(memnew(ParticlesGizmoPlugin))); + add_gizmo_plugin(Ref(memnew(CPUParticlesGizmoPlugin))); add_gizmo_plugin(Ref(memnew(ReflectionProbeGizmoPlugin))); add_gizmo_plugin(Ref(memnew(GIProbeGizmoPlugin))); add_gizmo_plugin(Ref(memnew(BakedIndirectLightGizmoPlugin))); diff --git a/editor/spatial_editor_gizmos.cpp b/editor/spatial_editor_gizmos.cpp index fa9d32f8eb2..85c1d74b0b9 100644 --- a/editor/spatial_editor_gizmos.cpp +++ b/editor/spatial_editor_gizmos.cpp @@ -36,6 +36,7 @@ #include "scene/3d/baked_lightmap.h" #include "scene/3d/collision_polygon.h" #include "scene/3d/collision_shape.h" +#include "scene/3d/cpu_particles.h" #include "scene/3d/gi_probe.h" #include "scene/3d/light.h" #include "scene/3d/listener.h" @@ -2375,6 +2376,33 @@ void VisibilityNotifierGizmoPlugin::redraw(EditorSpatialGizmo *p_gizmo) { //// +CPUParticlesGizmoPlugin::CPUParticlesGizmoPlugin() { + create_icon_material("particles_icon", SpatialEditor::get_singleton()->get_icon("GizmoCPUParticles", "EditorIcons")); +} + +bool CPUParticlesGizmoPlugin::has_gizmo(Spatial *p_spatial) { + return Object::cast_to(p_spatial) != NULL; +} + +String CPUParticlesGizmoPlugin::get_name() const { + return "CPUParticles"; +} + +int CPUParticlesGizmoPlugin::get_priority() const { + return -1; +} + +bool CPUParticlesGizmoPlugin::is_selectable_when_hidden() const { + return true; +} + +void CPUParticlesGizmoPlugin::redraw(EditorSpatialGizmo *p_gizmo) { + Ref icon = get_material("particles_icon", p_gizmo); + p_gizmo->add_unscaled_billboard(icon, 0.05); +} + +//// + ParticlesGizmoPlugin::ParticlesGizmoPlugin() { Color gizmo_color = EDITOR_DEF("editors/3d_gizmos/gizmo_colors/particles", Color(0.8, 0.7, 0.4)); create_material("particles_material", gizmo_color); diff --git a/editor/spatial_editor_gizmos.h b/editor/spatial_editor_gizmos.h index 3661df4bad3..317ea0c5705 100644 --- a/editor/spatial_editor_gizmos.h +++ b/editor/spatial_editor_gizmos.h @@ -249,6 +249,18 @@ public: VisibilityNotifierGizmoPlugin(); }; +class CPUParticlesGizmoPlugin : public EditorSpatialGizmoPlugin { + GDCLASS(CPUParticlesGizmoPlugin, EditorSpatialGizmoPlugin); + +public: + bool has_gizmo(Spatial *p_spatial); + String get_name() const; + int get_priority() const; + bool is_selectable_when_hidden() const; + void redraw(EditorSpatialGizmo *p_gizmo); + CPUParticlesGizmoPlugin(); +}; + class ParticlesGizmoPlugin : public EditorSpatialGizmoPlugin { GDCLASS(ParticlesGizmoPlugin, EditorSpatialGizmoPlugin); From 41d0e46f2da2dee1a7882029797e4ce0a643a3a9 Mon Sep 17 00:00:00 2001 From: Hugo Locurcio Date: Mon, 12 Aug 2019 22:30:57 +0200 Subject: [PATCH 107/113] Always use lists for `LIBS` in SCons This closes #31288. (cherry picked from commit 05daf5c78be6e6c2f6a74a129edd1c53826a9f8e) --- modules/mono/config.py | 6 +++--- platform/SCsub | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/mono/config.py b/modules/mono/config.py index 6b88600c565..df7bdbdaad9 100644 --- a/modules/mono/config.py +++ b/modules/mono/config.py @@ -120,8 +120,8 @@ def configure(env): else: env.Append(LINKFLAGS=os.path.join(mono_lib_path, mono_static_lib_name + lib_suffix)) - env.Append(LIBS='psapi') - env.Append(LIBS='version') + env.Append(LIBS=['psapi']) + env.Append(LIBS=['version']) else: mono_lib_name = find_file_in_dir(mono_lib_path, mono_lib_names, extension='.lib') @@ -131,7 +131,7 @@ def configure(env): if env.msvc: env.Append(LINKFLAGS=mono_lib_name + Environment()['LIBSUFFIX']) else: - env.Append(LIBS=mono_lib_name) + env.Append(LIBS=[mono_lib_name]) mono_bin_path = os.path.join(mono_root, 'bin') diff --git a/platform/SCsub b/platform/SCsub index aa83154ee0c..61324cfb063 100644 --- a/platform/SCsub +++ b/platform/SCsub @@ -28,4 +28,4 @@ with open_utf8('register_platform_apis.gen.cpp', 'w') as f: platform_sources.append('register_platform_apis.gen.cpp') lib = env.add_library('platform', platform_sources) -env.Prepend(LIBS=lib) +env.Prepend(LIBS=[lib]) From 6a4653f41cc8010734124c5791ec1011c1e19283 Mon Sep 17 00:00:00 2001 From: fogine <7884288+fogine@users.noreply.github.com> Date: Wed, 21 Aug 2019 18:55:21 +0200 Subject: [PATCH 108/113] iOS>=11 platform - when handling gestures on screen edges, godot apps should have priority over OS Solves an issue where iOS would steal InputEventTouch events when near screen edges in order to handle system wide gestures. Fixes #31503 (cherry picked from commit e0df9de0cb307b415e23a5157092eb5c8334c6b0) --- platform/iphone/view_controller.h | 4 ++++ platform/iphone/view_controller.mm | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/platform/iphone/view_controller.h b/platform/iphone/view_controller.h index fc18661f621..68e3bc64fc7 100644 --- a/platform/iphone/view_controller.h +++ b/platform/iphone/view_controller.h @@ -39,6 +39,10 @@ - (void)didReceiveMemoryWarning; +- (void)viewDidLoad; + +- (UIRectEdge)preferredScreenEdgesDeferringSystemGestures; + - (BOOL)prefersStatusBarHidden; @end diff --git a/platform/iphone/view_controller.mm b/platform/iphone/view_controller.mm index 0358abf9e23..e52ad92bf2a 100644 --- a/platform/iphone/view_controller.mm +++ b/platform/iphone/view_controller.mm @@ -83,6 +83,18 @@ int add_cmdline(int p_argc, char **p_args) { printf("*********** did receive memory warning!\n"); }; +- (void)viewDidLoad { + [super viewDidLoad]; + + if (@available(iOS 11.0, *)) { + [self setNeedsUpdateOfScreenEdgesDeferringSystemGestures]; + } +} + +- (UIRectEdge)preferredScreenEdgesDeferringSystemGestures { + return UIRectEdgeAll; +} + - (BOOL)shouldAutorotate { switch (OS::get_singleton()->get_screen_orientation()) { case OS::SCREEN_SENSOR: From c3aefbd1d2b508c3f8d708d9f8b410438923e9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 27 Aug 2019 10:19:07 +0200 Subject: [PATCH 109/113] Android: Bump gradle version to 5.1.1 Matching changes made in #31521 and #31547 but only in the Jetbrains IDE config. (cherry picked from commit 1afd77e375b39c39dde1d4a236ed073df9e8b6d8) --- platform/android/build.gradle.template | 2 +- platform/android/java/gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platform/android/build.gradle.template b/platform/android/build.gradle.template index 2fea2500614..27efc188d92 100644 --- a/platform/android/build.gradle.template +++ b/platform/android/build.gradle.template @@ -5,7 +5,7 @@ buildscript { $$GRADLE_REPOSITORY_URLS$$ } dependencies { - classpath 'com.android.tools.build:gradle:3.2.1' + classpath 'com.android.tools.build:gradle:3.4.2' $$GRADLE_CLASSPATH$$ } } diff --git a/platform/android/java/gradle/wrapper/gradle-wrapper.properties b/platform/android/java/gradle/wrapper/gradle-wrapper.properties index bf3de218305..558870dad58 100644 --- a/platform/android/java/gradle/wrapper/gradle-wrapper.properties +++ b/platform/android/java/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 8b4eea3d6d9aa53f9c38f8fcda773186a07b9369 Mon Sep 17 00:00:00 2001 From: Ignacio Etcheverry Date: Wed, 25 Sep 2019 22:02:24 +0200 Subject: [PATCH 110/113] Mono: Don't use project settings for debugger agent on exported games (cherry picked from commit 5ff4e0516b01f4bd1d5ffd80e35f8e9a623bca72) --- modules/mono/mono_gd/gd_mono.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/mono/mono_gd/gd_mono.cpp b/modules/mono/mono_gd/gd_mono.cpp index f70dad9c84b..fc3273500f0 100644 --- a/modules/mono/mono_gd/gd_mono.cpp +++ b/modules/mono/mono_gd/gd_mono.cpp @@ -121,25 +121,28 @@ void gdmono_debug_init() { mono_debug_init(MONO_DEBUG_FORMAT_MONO); + CharString da_args = OS::get_singleton()->get_environment("GODOT_MONO_DEBUGGER_AGENT").utf8(); + +#ifdef TOOLS_ENABLED int da_port = GLOBAL_DEF("mono/debugger_agent/port", 23685); bool da_suspend = GLOBAL_DEF("mono/debugger_agent/wait_for_debugger", false); int da_timeout = GLOBAL_DEF("mono/debugger_agent/wait_timeout", 3000); -#ifdef TOOLS_ENABLED if (Engine::get_singleton()->is_editor_hint() || ProjectSettings::get_singleton()->get_resource_path().empty() || Main::is_project_manager()) { return; } -#endif - - CharString da_args = OS::get_singleton()->get_environment("GODOT_MONO_DEBUGGER_AGENT").utf8(); if (da_args.length() == 0) { da_args = String("--debugger-agent=transport=dt_socket,address=127.0.0.1:" + itos(da_port) + ",embedding=1,server=y,suspend=" + (da_suspend ? "y,timeout=" + itos(da_timeout) : "n")) .utf8(); } +#else + if (da_args.length() == 0) + return; // Exported games don't use the project settings to setup the debugger agent +#endif // --debugger-agent=help const char *options[] = { From c1cabb0bf554f31df00ce3925a23cffade4de280 Mon Sep 17 00:00:00 2001 From: Yuri Roubinsky Date: Sun, 6 Oct 2019 20:35:41 +0300 Subject: [PATCH 111/113] Prevent shader crash if name of variable overrides function name (cherry picked from commit 76324bec8d269330c59812bfcc852a195304aa92) --- servers/visual/shader_language.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/servers/visual/shader_language.cpp b/servers/visual/shader_language.cpp index 9ab7206c75b..db9173e7b0c 100644 --- a/servers/visual/shader_language.cpp +++ b/servers/visual/shader_language.cpp @@ -2848,6 +2848,16 @@ ShaderLanguage::Node *ShaderLanguage::_parse_expression(BlockNode *p_block, cons bool ok = _parse_function_arguments(p_block, p_builtin_types, func, &carg); + // Check if block has a variable with the same name as function to prevent shader crash. + ShaderLanguage::BlockNode *bnode = p_block; + while (bnode) { + if (bnode->variables.has(name)) { + _set_error("Expected function name"); + return NULL; + } + bnode = bnode->parent_block; + } + //test if function was parsed first for (int i = 0; i < shader->functions.size(); i++) { if (shader->functions[i].name == name) { From 7e56ef1484468f912582cf3fcb6704053378789f Mon Sep 17 00:00:00 2001 From: Marcel Admiraal Date: Sat, 12 Oct 2019 08:02:03 +0200 Subject: [PATCH 112/113] Fix mismatch between RigidBody(2D) and Body(2D)SW can_sleep defaults. - Set Body2DSW can_sleep default to true. - Set Body2D can_sleep default to true. (cherry picked from commit 42a9ae7a82839fa4f69281f247a26091b612aa8e) --- servers/physics/body_sw.cpp | 2 +- servers/physics_2d/body_2d_sw.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/servers/physics/body_sw.cpp b/servers/physics/body_sw.cpp index f0fbbafe1c9..a60f548d34f 100644 --- a/servers/physics/body_sw.cpp +++ b/servers/physics/body_sw.cpp @@ -791,7 +791,7 @@ BodySW::BodySW() : still_time = 0; continuous_cd = false; - can_sleep = false; + can_sleep = true; fi_callback = NULL; } diff --git a/servers/physics_2d/body_2d_sw.cpp b/servers/physics_2d/body_2d_sw.cpp index 60bcb2e28e0..a103688deaa 100644 --- a/servers/physics_2d/body_2d_sw.cpp +++ b/servers/physics_2d/body_2d_sw.cpp @@ -687,7 +687,7 @@ Body2DSW::Body2DSW() : still_time = 0; continuous_cd_mode = Physics2DServer::CCD_MODE_DISABLED; - can_sleep = false; + can_sleep = true; fi_callback = NULL; } From 473e3665ff51d43eb481e55024d21171c584fcd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Tue, 12 Nov 2019 14:29:41 +0100 Subject: [PATCH 113/113] Bump version to 3.1.2-rc --- modules/mono/doc_classes/@C#.xml | 2 +- modules/mono/doc_classes/CSharpScript.xml | 2 +- modules/mono/doc_classes/GodotSharp.xml | 2 +- version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/mono/doc_classes/@C#.xml b/modules/mono/doc_classes/@C#.xml index 035c5454676..dda23020702 100644 --- a/modules/mono/doc_classes/@C#.xml +++ b/modules/mono/doc_classes/@C#.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/mono/doc_classes/CSharpScript.xml b/modules/mono/doc_classes/CSharpScript.xml index a9ff32d8e24..e77a1319a18 100644 --- a/modules/mono/doc_classes/CSharpScript.xml +++ b/modules/mono/doc_classes/CSharpScript.xml @@ -1,5 +1,5 @@ - + diff --git a/modules/mono/doc_classes/GodotSharp.xml b/modules/mono/doc_classes/GodotSharp.xml index 088b1b9f822..414d5acb1e1 100644 --- a/modules/mono/doc_classes/GodotSharp.xml +++ b/modules/mono/doc_classes/GodotSharp.xml @@ -1,5 +1,5 @@ - + diff --git a/version.py b/version.py index f0958d0ba2b..aed645f0e06 100644 --- a/version.py +++ b/version.py @@ -3,7 +3,7 @@ name = "Godot Engine" major = 3 minor = 1 patch = 2 -status = "devel" +status = "rc" module_config = "" year = 2019 website = "https://godotengine.org"