Merged with develop

CHANGES_NEXT_RELEASE

@@ -2,6 +2,7 @@
   * #1535:  Ignore Subscription::q when matching subscriptions for deletion of an entity
   * #1542:  Array Reduction (arrays of one single element are "flattened" to that very element)
   * SOF:    https://stackoverflow.com/questions/77911109/fiware-to-payload-too-large
+  * #1551:  format=simplified for distribute GET /entities gave Normalized for entities from other brokers
 
 ## New Features:
   * Support for attributes of type VocabularyProperty
@@ -12,3 +13,7 @@
 
 ## Notes
   * TRoE is still not prepared for attributes of type Vocab/Json/Language, so, attributes of those types are not stored in the historical database
+  * Modified the @context hosting feature to be according to API spec
+  * Support for URI param 'kind' for GET Contexts
+  * Improved counters for GET Context/Contexts
+  * API spec compliance for GET Context/Contexts

src/app/orionld/orionld.cpp

@@ -243,6 +243,7 @@ bool            debugCurl    = false;
 uint32_t        cSubCounters;
 char            coreContextVersion[64];
 bool            triggerOperation = false;
+bool            noprom           = false;
 
 
 
@@ -335,6 +336,7 @@ bool            triggerOperation = false;
 #define DEBUG_CURL_DESC        "turn on debugging of libcurl - to the broker's logfile"
 #define CSUBCOUNTERS_DESC      "number of subscription counter updates before flush from sub-cache to DB (0: never, 1: always)"
 #define CORE_CONTEXT_DESC      "core context version (v1.0|v1.3|v1.4|v1.5|v1.6|v1.7) - v1.6 is default"
+#define NO_PROM_DESC           "run without Prometheus metrics"
 
 
 
@@ -438,6 +440,7 @@ PaArgument paArgs[] =
   { "-lmtmp",                 &lmtmp,                   "TMP_TRACES",                PaBool,    PaHid,  true,            false,  true,             TMPTRACES_DESC           },
   { "-debugCurl",             &debugCurl,               "DEBUG_CURL",                PaBool,    PaHid,  false,           false,  true,             DEBUG_CURL_DESC          },
   { "-lmtmp",                 &lmtmp,                   "TMP_TRACES",                PaBool,    PaHid,  true,            false,  true,             TMPTRACES_DESC           },
+  { "-noprom",                &noprom,                  "NO_PROM",                   PaBool,    PaHid,  false,           false,  true,             NO_PROM_DESC             },
 
   PA_END_OF_ARGS
 };
@@ -1130,7 +1133,9 @@ int main(int argC, char* argV[])
   if (fg == false)
     daemonize();
 
-  if (promInit(8000) != 0)
+  if (noprom == true)
+    LM_W(("Running without Prometheus metrics"));
+  else if (promInit(8000) != 0)
     LM_W(("Error initializing Prometheus Metrics library"));
 
   IpVersion ipVersion = IPDUAL;

src/lib/logMsg/traceLevels.h

@@ -101,6 +101,7 @@ typedef enum TraceLevels
   LmtContexts = 100,                   // Contexts
   LmtContextTree,                      // Context Tree
   LmtContextCache,                     // Context Cache
+  LmtContextCacheStats,                // Context Cache Statistics
   LmtContextDownload,                  // Context Download
   LmtCoreContext,                      // Core Context
 
@@ -142,6 +143,7 @@ typedef enum TraceLevels
   LmtDateTime,                         // DateTime (ISO8601) conversion
   LmtMimeType,                         // MimeType
   LmtArrayReduction,                   // Arrays of only one item are reduced to the item
+  LmtFormat,                           // Normalized, Concise, Simplified
 
   //
   // Legacy

src/lib/orionld/apiModel/ntocAttribute.cpp

@@ -57,6 +57,8 @@ void ntocAttribute(KjNode* attrP, const char* lang, bool sysAttrs)
     return;
   }
 
+  kjTreeLog(attrP, "attribute to convert from Normalized to Concise", LmtFormat);
+
   // 1. Remove the sysAttrs, if so requested
   if (sysAttrs == false)
     kjSysAttrsRemove(attrP, 0);

src/lib/orionld/common/orionldState.cpp

@@ -118,6 +118,7 @@ bool              entityMapsEnabled = false;
 mongoc_uri_t*          mongocUri;
 mongoc_client_pool_t*  mongocPool;
 sem_t                  mongocContextsSem;
+sem_t                  mongocConnectionSem;
 char                   mongocServerVersion[128];
 char                   postgresServerVersion[128];
 

src/lib/orionld/common/orionldState.h

@@ -170,6 +170,8 @@ typedef struct OrionldUriParams
   bool      entityMap;
   char*     format;
 
+  OrionldContextKind kind;
+
   double    observedAtAsDouble;
   uint64_t  mask;
 } OrionldUriParams;
@@ -639,6 +641,7 @@ extern mongoc_collection_t*  mongoRegistrationsCollectionP;  // Deprecated
 extern mongoc_uri_t*          mongocUri;
 extern mongoc_client_pool_t*  mongocPool;
 extern sem_t                  mongocContextsSem;
+extern sem_t                  mongocConnectionSem;
 extern char                   mongocServerVersion[128];
 extern char                   postgresServerVersion[128];
 

src/lib/orionld/context/CMakeLists.txt

@@ -42,7 +42,6 @@ SET (SOURCES
     orionldContextItemAlreadyExpanded.cpp
     orionldAttributeExpand.cpp
     orionldSubAttributeExpand.cpp
-    orionldContextOriginName.cpp
     orionldEntityExpand.cpp
     orionldEntityCompact.cpp
 )

src/lib/orionld/context/orionldContextCreate.cpp

@@ -37,25 +37,6 @@ extern "C"
 
 
 
-// -----------------------------------------------------------------------------
-//
-// orionldOriginFromString - FIXME: move to its own module
-//
-OrionldContextOrigin orionldOriginFromString(const char* s)
-{
-  if      (strcmp(s, "UnknownOrigin")    == 0) return OrionldContextUnknownOrigin;
-  else if (strcmp(s, "Inline")           == 0) return OrionldContextFromInline;
-  else if (strcmp(s, "Downloaded")       == 0) return OrionldContextDownloaded;
-  else if (strcmp(s, "FileCached")       == 0) return OrionldContextFileCached;
-  else if (strcmp(s, "ForNotifications") == 0) return OrionldContextForNotifications;
-  else if (strcmp(s, "ForForwarding")    == 0) return OrionldContextForForwarding;
-  else if (strcmp(s, "UserCreated")      == 0) return OrionldContextUserCreated;
-
-  return OrionldContextUnknownOrigin;
-}
-
-
-
 // -----------------------------------------------------------------------------
 //
 // orionldContextCreate -
@@ -72,6 +53,7 @@ OrionldContext* orionldContextCreate(const char* url, OrionldContextOrigin origi
     LM_X(1, ("out of memory - trying to allocate a OrionldContext of %d bytes", sizeof(OrionldContext)));
 
   contextP->origin    = origin;
+  contextP->kind      = OrionldContextCached;  // By defaukt. Changed later to Hosted/Implicit if needed
   contextP->parent    = NULL;
 
   // NULL URL means NOT to be saved - will live just inside the request-thread
@@ -95,7 +77,7 @@ OrionldContext* orionldContextCreate(const char* url, OrionldContextOrigin origi
   }
 
   contextP->keyValues = keyValues;
-  contextP->lookups   = 0;
+  contextP->lookups   = 1;
 
   return contextP;
 }

src/lib/orionld/context/orionldContextFromBuffer.cpp

@@ -67,7 +67,7 @@ OrionldContext* orionldContextFromBuffer(char* url, OrionldContextOrigin origin,
   KjNode* contextNodeP = kjLookup(tree, "@context");
   if (contextNodeP == NULL)
   {
-    orionldError(OrionldBadRequestData, "Invalid context - @context field missing", url, 400);
+    orionldError(OrionldInvalidRequest, "Invalid context - @context field missing", url, 400);
     return NULL;
   }
 

src/lib/orionld/context/orionldContextFromUrl.cpp

@@ -251,6 +251,11 @@ OrionldContext* orionldContextFromUrl(char* url, char* id)
 
   if (contextP != NULL)
   {
+    contextP->usedAt   = orionldState.requestTime;
+
+    contextP->lookups += 1;
+    LM_T(LmtContextCacheStats, ("Context '%s': %d lookups", url, contextP->lookups));
+
     LM_T(LmtContextDownload, ("Found already downloaded URL '%s'", url));
     return contextP;
   }

src/lib/orionld/context/orionldContextItemAliasLookup.cpp

@@ -59,10 +59,15 @@ char* orionldContextItemAliasLookup
   if (contextItemPP != NULL)
     *contextItemPP = NULL;
 
+  if (contextP == NULL)
+    contextP = orionldCoreContextP;
 
   // 1. Is it the default URL?
   if (strncmp(longName, orionldDefaultUrl, orionldDefaultUrlLen) == 0)
+  {
+    orionldCoreContextP->compactions += 1;
     return (char*) &longName[orionldDefaultUrlLen];
+  }
 
   // 2. Found in Core Context?
   contextItemP = orionldContextItemValueLookup(orionldCoreContextP, longName);
@@ -91,5 +96,6 @@ char* orionldContextItemAliasLookup
     *contextItemPP = contextItemP;
 
   // Return the short name
+  contextP->compactions += 1;
   return contextItemP->name;
 }

src/lib/orionld/context/orionldContextItemExpand.cpp

@@ -96,6 +96,8 @@ char* orionldContextItemExpand
       if (contextItemPP != NULL)
         *contextItemPP = NULL;
 
+      orionldCoreContextP->expansions += 1;  // Really, @vocab expansions of the core context
+
       return longName;
     }
 
@@ -106,5 +108,7 @@ char* orionldContextItemExpand
   if (contextItemPP != NULL)
     *contextItemPP = contextItemP;
 
+  contextP->expansions += 1;  // Really, @vocab expansions of the core context
+
   return contextItemP->id;
 }

src/lib/orionld/context/orionldContextItemLookup.cpp

@@ -62,12 +62,17 @@ OrionldContextItem* orionldContextItemLookup(OrionldContext* contextP, const cha
     }
   }
 
-  if (valueMayBeCompactedP != NULL)
+  if (itemP != NULL)
   {
-    if ((itemP->type != NULL) && (strcmp(itemP->type, "@vocab") == 0))
-      *valueMayBeCompactedP = true;
-    else
-      *valueMayBeCompactedP = false;
+    contextP->expansions += 1;
+
+    if (valueMayBeCompactedP != NULL)
+    {
+      if ((itemP->type != NULL) && (strcmp(itemP->type, "@vocab") == 0))
+        *valueMayBeCompactedP = true;
+      else
+        *valueMayBeCompactedP = false;
+    }
   }
 
   return itemP;