Skip to content

Commit

Permalink
Merge branch 'develop' into attribute/json
Browse files Browse the repository at this point in the history
  • Loading branch information
kzangeli authored Feb 8, 2024
2 parents 3d9d453 + da204f8 commit da656b7
Show file tree
Hide file tree
Showing 7 changed files with 173 additions and 17 deletions.
8 changes: 5 additions & 3 deletions CHANGES_NEXT_RELEASE
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
## Fixed Issues:
* #1535: No matching with 'q' when matching subscriptions for deletion of an entity
* #1542: Array Reduction (arrays of one single element are "flattened" to that very element)
* #1535: Ignore Subscription::q when matching subscriptions for deletion of an entity
* #1542: Array Reduction (arrays of one single element are "flattened" to that very element)
* SOF: https://stackoverflow.com/questions/77911109/fiware-to-payload-too-large

## New Features:
* Support for VocabularyProperty
* Support for attributes of type VocabularyProperty
* Support for attributes of type JsonProperty
* Support for the new URL parameter "format" for output formats (normalized, concise, simplified)
* Support for JsonProperty attribute type
* Fixed a possible crash for TRoE and attributes of type Vocab/Json/Language
Expand Down
11 changes: 10 additions & 1 deletion src/lib/orionld/mhd/mhdConnectionInit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,16 @@ static MHD_Result orionldHttpHeaderReceive(void* cbDataP, MHD_ValueKind kind, co
else if (strcasecmp(key, "X-Auth-Token") == 0) orionldState.in.xAuthToken = (char*) value;
else if (strcasecmp(key, "Authorization") == 0) orionldState.in.authorization = (char*) value;
else if (strcasecmp(key, "Fiware-Correlator") == 0) orionldState.correlator = (char*) value;
else if (strcasecmp(key, "Content-Length") == 0) orionldState.in.contentLength = atoi(value);
else if (strcasecmp(key, "Content-Length") == 0)
{
orionldState.in.contentLength = atoi(value);
if ((unsigned long long) orionldState.in.contentLength > inReqPayloadMaxSize)
{
char detail[256];
snprintf(detail, sizeof(detail), "payload size: %u, max size supported: %llu", orionldState.in.contentLength, inReqPayloadMaxSize);
orionldError(OrionldBadRequestData, "Request Entity too large", detail, 413);
}
}
else if (strcasecmp(key, "Prefer") == 0) orionldState.preferHeader = (char*) value;
else if (strcasecmp(key, "Origin") == 0) orionldState.in.origin = (char*) value;
else if (strcasecmp(key, "Host") == 0) orionldState.in.host = (char*) value;
Expand Down
2 changes: 1 addition & 1 deletion src/lib/orionld/mhd/mhdConnectionTreat.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ static bool payloadParseAndExtractSpecialFields(bool* contextToBeCashedP)
//
if (orionldState.requestTree == NULL)
{
orionldError(OrionldInvalidRequest, "JSON Parse Error", orionldState.kjsonP->errorString, 400);
orionldError(OrionldInvalidRequest, "JSON Parse Error", "Invalid JSON payload", 400);
return false;
}

Expand Down
18 changes: 9 additions & 9 deletions test/functionalTest/cases/0000_ngsild/ngsild_context_errors.test
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ orionldStart CB
# 10. GET Contexts - see only Core Context
# 11. URL in context is a valid URL but it doesn't exist
# 12. GET Contexts - see only Core Context
# 13. Ugly Parse Error: @context: { 'https://...' ]
# 13. Ugly Parse Error: @context: { 'http://...' ]
# 14. GET Contexts - see only Core Context
# 15. Parse Error in an external context (https://fiware.github.io/NGSI-LD_TestSuite/ldContext/contextWithParseError.jsonld)
# 16. GET Contexts - see only Core Context
Expand Down Expand Up @@ -200,8 +200,8 @@ echo
echo


echo "13. Ugly Parse Error: @context: { 'https://...' ]"
echo "================================================="
echo "13. Ugly Parse Error: @context: { 'http://...' ]"
echo "================================================"
payload='{
"id": "urn:E1",
"type": "A",
Expand Down Expand Up @@ -531,15 +531,15 @@ Date: REGEX(.*)
]


13. Ugly Parse Error: @context: { 'https://...' ]
=================================================
13. Ugly Parse Error: @context: { 'http://...' ]
================================================
HTTP/1.1 400 Bad Request
Content-Length: 153
Content-Length: 120
Content-Type: application/json
Date: REGEX(.*)

{
"detail": "JSON Parse Error: no colon found after name of member",
"detail": "Invalid JSON payload",
"title": "JSON Parse Error",
"type": "https://uri.etsi.org/ngsi-ld/errors/InvalidRequest"
}
Expand Down Expand Up @@ -651,12 +651,12 @@ Date: REGEX(.*)
19. Inline context that is an array of three and the middle member is a context that gives parse error
======================================================================================================
HTTP/1.1 400 Bad Request
Content-Length: 149
Content-Length: 120
Content-Type: application/json
Date: REGEX(.*)

{
"detail": "JSON Parse Error: expecting comma or end of array",
"detail": "Invalid JSON payload",
"title": "JSON Parse Error",
"type": "https://uri.etsi.org/ngsi-ld/errors/InvalidRequest"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ echo
01. POST /ngsi-ld/v1/entities, with a parse error in the incoming payload (missing comma after 'Property')
==========================================================================================================
HTTP/1.1 400 Bad Request
Content-Length: 150
Content-Length: 120
Content-Type: application/json
Date: REGEX(.*)

{
"detail": "JSON Parse Error: expecting comma or end of object",
"detail": "Invalid JSON payload",
"title": "JSON Parse Error",
"type": "https://uri.etsi.org/ngsi-ld/errors/InvalidRequest"
}
Expand Down
145 changes: 145 additions & 0 deletions test/functionalTest/cases/0000_ngsild/ngsild_large-payloads.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# Copyright 2024 FIWARE Foundation e.V.
#
# This file is part of Orion-LD Context Broker.
#
# Orion-LD Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion-LD Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion-LD Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# orionld at fiware dot org

# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh

--NAME--
Large requests - starting the broker to accept requetsts of up to 4Mb (more or less)

--SHELL-INIT--
dbInit CB
orionldStart CB -inReqPayloadMaxSize 4000000 -experimental

--SHELL--
#
# Creating a file of ~5 MB.
# The default max size of a request is 1 MB but the broker is started with -inReqPayloadMaxSize 4000000 (~4 MB), and a ~3 MB request must be permitted
#
#
# Note that we are not using a valid JSON payload - in this test case, what matters is the size, not the format of the data.
#
# 01. As the HUGE payload is accepted, size-wise, it will be parsed and a Parse Error will be the result.
# 02. In test case 02, the HUGE payload is too big and is not accepted - a "413 Content Too Large" is returned
# 03. Creating an entity with 1000 Attributes of 3k each - the total payload body is well over 3M, but as the broker accepts close to 4M, all OK
# 04. GET the entity from step 03 to make sure all is good
#

echo "01. Sending a request of ~3 MB, to a broker that excepts request upto ~4 MB - see parse error as a result"
echo "========================================================================================================="
tr -dc A-Za-z0-9 < /dev/urandom 2>/dev/null | head -c 3000000 > /tmp/threeMegas
orionCurl --url /ngsi-ld/v1/entities --payload /tmp/threeMegas
echo
echo


echo "02. Sending a request of ~5 MB, to a broker that excepts request upto ~4 MB - see 413 error"
echo "==========================================================================================="
tr -dc A-Za-z0-9 < /dev/urandom 2>/dev/null | head -c 5000000 > /tmp/fiveMegas
orionCurl --url /ngsi-ld/v1/entities --payload /tmp/fiveMegas
echo
echo


echo "03. Creating an entity with 1000 Attributes of 3k each - the total payload body is well over 3M, but as the broker accepts close to 4M, all OK"
echo "=============================================================================================================================================="
echo '{' > /tmp/entity
echo ' "id": "urn:E1",' >> /tmp/entity
echo ' "type": "T",' >> /tmp/entity

ATTRS=1000
ATTRSIZE=3072
attrValue=$(tr -dc A-Za-z < /dev/urandom 2>/dev/null | head -c $ATTRSIZE)

typeset -i aIx
aIx=1
while [ $aIx -le $ATTRS ]
do
attr='"A'$aIx'": { "type": "Property", "value": "'$attrValue'"}'
if [ $aIx -lt $ATTRS ]
then
attr=$attr','
fi
echo ' '$attr >> /tmp/entity
aIx=$aIx+1
done
echo '}' >> /tmp/entity

orionCurl --url /ngsi-ld/v1/entities --payload /tmp/entity
echo
echo


echo "04. GET the entity from step 03 to make sure all is good"
echo "========================================================"
orionCurl --url /ngsi-ld/v1/entities/urn:E1 | head -5
echo
echo


--REGEXPECT--
01. Sending a request of ~3 MB, to a broker that excepts request upto ~4 MB - see parse error as a result
=========================================================================================================
HTTP/1.1 400 Bad Request
Content-Length: 120
Content-Type: application/json
Date: REGEX(.*)

{
"detail": "Invalid JSON payload",
"title": "JSON Parse Error",
"type": "https://uri.etsi.org/ngsi-ld/errors/InvalidRequest"
}


02. Sending a request of ~5 MB, to a broker that excepts request upto ~4 MB - see 413 error
===========================================================================================
HTTP/1.1 413 Content Too Large
Content-Length: 158
Content-Type: application/json
Date: REGEX(.*)

{
"detail": "payload size: 5000000, max size supported: 4000000",
"title": "Request Entity too large",
"type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData"
}


03. Creating an entity with 1000 Attributes of 3k each - the total payload body is well over 3M, but as the broker accepts close to 4M, all OK
==============================================================================================================================================
HTTP/1.1 201 Created
Content-Length: 0
Date: REGEX(.*)
Location: /ngsi-ld/v1/entities/urn:E1



04. GET the entity from step 03 to make sure all is good
========================================================
HTTP/1.1 200 OK
Content-Length: 3109919
Content-Type: application/json
Date: REGEX(.*)
Link: <https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-contextREGEX(.*)


--TEARDOWN--
brokerStop CB
dbDrop CB
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ Large request - starting the broker to accept very large requests

--SHELL-INIT--
dbInit CB
brokerStart CB 0-255 IPV4 -inReqPayloadMaxSize 50000000
brokerStart CB 0-255 IPV4 -inReqPayloadMaxSize 50000000 -experimental

--SHELL--
#
Expand Down

0 comments on commit da656b7

Please sign in to comment.