Merge remote-tracking branch 'origin/main' into INTG-2294-error-durin…

…g-send-text-call-after-capability-changing

.github/workflows/reviewdog.yml

@@ -42,3 +42,14 @@ jobs:
       - name: Linelint
         uses: fernandrone/linelint@master
         id: linelint
+
+  circular-deps:
+    runs-on: ubuntu-latest
+    name: Check for circular dependencies
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build node env
+        uses: ./.github/actions/build_package
+      - name: Check for circular dependencies
+        run: yarn circular-deps

package.json

@@ -22,7 +22,8 @@
     "prettier:format": "yarn workspaces foreach -A run prettier:format && yarn prettier --write \"./examples/**/*.{js,jsx,ts,tsx}\" --ignore-path .eslintignore --config .prettierrc.json",
     "prepare": "husky",
     "test": "yarn workspace @inworld/web-core run test",
-    "test:coverage": "yarn workspace @inworld/web-core run test:coverage"
+    "test:coverage": "yarn workspace @inworld/web-core run test:coverage",
+    "circular-deps": "yarn workspaces foreach -A run circular-deps"
   },
   "devDependencies": {
     "@release-it/keep-a-changelog": "^5.0.0",
@@ -39,6 +40,7 @@
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "jest-websocket-mock": "^2.5.0",
+    "madge": "^8.0.0",
     "prettier": "^3.3.2",
     "release-it": "^17.1.1",
     "rollup": "^4.16.1",

packages/web-core/__tests__/components/history.spec.ts

@@ -3,20 +3,22 @@ import '../mocks/window.mock';
 import { v4 } from 'uuid';
 
 import {
-  ConversationMapItem,
   ConversationState,
-  Extension,
   InworlControlAction,
   InworldConversationEventType,
   InworldPacketType,
   User,
 } from '../../src/common/data_structures';
-import { protoTimestamp } from '../../src/common/helpers';
+import {
+  ConvesationInterface,
+  Extension,
+} from '../../src/common/data_structures/extension';
 import {
   CHAT_HISTORY_TYPE,
   HistoryItemActor,
-  InworldHistory,
-} from '../../src/components/history';
+} from '../../src/common/data_structures/history';
+import { protoTimestamp } from '../../src/common/helpers';
+import { InworldHistory } from '../../src/components/history';
 import { GrpcAudioPlayback } from '../../src/components/sound/grpc_audio.playback';
 import { ControlEvent } from '../../src/entities/packets/control.entity';
 import { InworldPacket } from '../../src/entities/packets/inworld_packet.entity';
@@ -113,7 +115,13 @@ const createHistoryWithPacket = (
     user,
     scene: SCENE,
     audioEnabled,
-    conversations: new Map<string, ConversationMapItem>(),
+    conversations: new Map<
+      string,
+      {
+        service: ConvesationInterface;
+        state: ConversationState;
+      }
+    >(),
   });
 
   history.addOrUpdate({ characters, grpcAudioPlayer, packet, fromHistory });
@@ -125,7 +133,13 @@ test('should be empty by default', () => {
   const history = new InworldHistory({
     scene: SCENE,
     audioEnabled: false,
-    conversations: new Map<string, ConversationMapItem>(),
+    conversations: new Map<
+      string,
+      {
+        service: ConvesationInterface;
+        state: ConversationState;
+      }
+    >(),
   });
 
   expect(history.get().length).toEqual(0);
@@ -371,7 +385,13 @@ describe('text', () => {
       const history = new InworldHistory({
         scene: SCENE,
         audioEnabled: true,
-        conversations: new Map<string, ConversationMapItem>(),
+        conversations: new Map<
+          string,
+          {
+            service: ConvesationInterface;
+            state: ConversationState;
+          }
+        >(),
       });
 
       const transcript = history.getTranscript();
@@ -807,7 +827,13 @@ describe('conversation', () => {
         },
       }),
     });
-    const conversations = new Map<string, ConversationMapItem>();
+    const conversations = new Map<
+      string,
+      {
+        service: ConvesationInterface;
+        state: ConversationState;
+      }
+    >();
     conversations.set(conversationId, {
       service: new ConversationService(new ConnectionService(), {
         participants: [characters[0].resourceName],

packages/web-core/__tests__/helpers/extension.ts

@@ -1,5 +1,5 @@
 import { InworldPacket as ProtoPacket } from '../../proto/ai/inworld/packets/packets.pb';
-import { Extension } from '../../src/common/data_structures';
+import { Extension } from '../../src/common/data_structures/extension';
 import { InworldPacket } from '../../src/entities/packets/inworld_packet.entity';
 import { ExtendedHistoryItem, ExtendedInworldPacket } from '../data_structures';
 

packages/web-core/__tests__/services/connection.service.spec.ts

@@ -12,21 +12,22 @@ import {
 } from '../../proto/ai/inworld/packets/packets.pb';
 import {
   ConversationState,
-  HistoryChangedProps,
   LoadedScene,
 } from '../../src/common/data_structures';
-import { protoTimestamp } from '../../src/common/helpers';
 import {
   CHAT_HISTORY_TYPE,
+  HistoryChangedProps,
   HistoryItem,
-  InworldHistory,
-} from '../../src/components/history';
+} from '../../src/common/data_structures/history';
+import { protoTimestamp } from '../../src/common/helpers';
+import { InworldHistory } from '../../src/components/history';
 import { GrpcAudioPlayback } from '../../src/components/sound/grpc_audio.playback';
 import { GrpcWebRtcLoopbackBiDiSession } from '../../src/components/sound/grpc_web_rtc_loopback_bidi.session';
 import { Player } from '../../src/components/sound/player';
 import { WebSocketConnection } from '../../src/connection/web-socket.connection';
 import { Character } from '../../src/entities/character.entity';
 import { InworldPacket } from '../../src/entities/packets/inworld_packet.entity';
+import { PingPongType } from '../../src/entities/packets/latency/ping_pong_report_type.entity';
 import { Actor } from '../../src/entities/packets/routing.entity';
 import { SessionToken } from '../../src/entities/session_token.entity';
 import { EventFactory } from '../../src/factories/event';
@@ -249,7 +250,7 @@ describe('open', () => {
     jest.spyOn(Player.prototype, 'setStream').mockImplementation(jest.fn());
   });
 
-  test('should execute without errors123', async () => {
+  test('should execute without errors', async () => {
     const openSession = jest
       .spyOn(WebSocketConnection.prototype, 'openSession')
       .mockImplementationOnce(() =>
@@ -1023,6 +1024,123 @@ describe('onWarning', () => {
   });
 });
 
+describe('latency', () => {
+  const HOSTNAME = 'localhost:1234';
+
+  let connection: ConnectionService;
+  let server: WS;
+
+  beforeEach(() => {
+    server = new WS(`wss://${HOSTNAME}/v1/session/open`, {
+      jsonProtocol: true,
+    });
+
+    connection = new ConnectionService({
+      name: SCENE,
+      config: {
+        connection: { gateway: { hostname: HOSTNAME } },
+        capabilities: capabilitiesProps,
+      },
+      user,
+      onError,
+      onMessage,
+      onDisconnect,
+      onInterruption,
+      grpcAudioPlayer,
+      generateSessionToken,
+      webRtcLoopbackBiDiSession,
+    });
+
+    jest.spyOn(Player.prototype, 'setStream').mockImplementation(jest.fn());
+    jest
+      .spyOn(ConversationService.prototype, 'getConversationId')
+      .mockImplementation(() => conversationId);
+  });
+
+  afterEach(() => {
+    server.close();
+    WS.clean();
+    connection.close();
+  });
+
+  test('should receive ping and send pong event', async () => {
+    const pong = jest.spyOn(EventFactory.prototype, 'pong');
+    const packetId = {
+      packetId: v4(),
+      interactionId: v4(),
+      utteranceId: v4(),
+      correlationId: v4(),
+    };
+    const pingTimestamp = protoTimestamp();
+
+    await Promise.all([
+      connection.open(),
+      setTimeout(() => new Promise(emitSceneStatusEvent(server)), 0),
+    ]);
+
+    await server.connected;
+
+    server.send({
+      result: {
+        packetId,
+        routing: {
+          source: { type: ActorType.WORLD },
+          target: { type: ActorType.PLAYER },
+        },
+        latencyReport: {
+          pingPong: {
+            pingTimestamp,
+            type: PingPongType.PING,
+          },
+        },
+      },
+    });
+
+    expect(pong).toHaveBeenCalledTimes(1);
+    expect(pong).toHaveBeenCalledWith(packetId, pingTimestamp);
+  });
+
+  test('should send perceived latency event', async () => {
+    connection.conversations.set(conversationId, {
+      service: new ConversationService(connection, {
+        participants: [characters[0].resourceName],
+        conversationId,
+        addCharacters: jest.fn(),
+      }),
+      state: ConversationState.ACTIVE,
+    });
+
+    const perceivedLatency = jest.spyOn(
+      EventFactory.prototype,
+      'perceivedLatency',
+    );
+
+    await Promise.all([
+      connection.open(),
+      setTimeout(() => new Promise(emitSceneStatusEvent(server)), 0),
+    ]);
+
+    await server.connected;
+
+    const textRequest = eventFactory.text(v4(), { conversationId });
+    const textResponse = eventFactory.text(v4(), { conversationId });
+    textResponse.packetId = {
+      ...textRequest.packetId,
+      packetId: v4(),
+    };
+    textResponse.routing!.source = { type: ActorType.WORLD };
+
+    await connection.send(() => textRequest);
+
+    server.send({
+      result: textResponse,
+    });
+
+    expect(perceivedLatency).toHaveBeenCalledTimes(1);
+    expect(perceivedLatency).toHaveBeenCalledWith(0, 0);
+  });
+});
+
 describe('load scene', () => {
   test("should load scene if it's required", async () => {
     const setCurrentCharacter = jest.spyOn(

packages/web-core/__tests__/services/inworld_connection.service.spec.ts

@@ -11,24 +11,24 @@ import {
 } from '../../proto/ai/inworld/packets/packets.pb';
 import {
   AudioSessionState,
-  ConversationMapItem,
   ConversationState,
   InworldPacketType,
   LoadedScene,
   MicrophoneMode,
   TtsPlaybackAction,
   UnderstandingMode,
 } from '../../src/common/data_structures';
+import { ConvesationInterface } from '../../src/common/data_structures/extension';
+import {
+  CHAT_HISTORY_TYPE,
+  HistoryItem,
+} from '../../src/common/data_structures/history';
 import {
   CHARACTER_HAS_INVALID_FORMAT,
   SCENE_HAS_INVALID_FORMAT,
 } from '../../src/common/errors';
 import { protoTimestamp } from '../../src/common/helpers';
-import {
-  CHAT_HISTORY_TYPE,
-  HistoryItem,
-  InworldHistory,
-} from '../../src/components/history';
+import { InworldHistory } from '../../src/components/history';
 import { GrpcAudioPlayback } from '../../src/components/sound/grpc_audio.playback';
 import { GrpcAudioRecorder } from '../../src/components/sound/grpc_audio.recorder';
 import { GrpcWebRtcLoopbackBiDiSession } from '../../src/components/sound/grpc_web_rtc_loopback_bidi.session';
@@ -176,7 +176,13 @@ describe('history', () => {
     const history = new InworldHistory({
       scene: SCENE,
       audioEnabled: true,
-      conversations: new Map<string, ConversationMapItem>(),
+      conversations: new Map<
+        string,
+        {
+          service: ConvesationInterface;
+          state: ConversationState;
+        }
+      >(),
     });
     const packetId = getPacketId();
     const routing: Routing = {

packages/web-core/__tests__/services/session_state.service.spec.ts

@@ -3,8 +3,11 @@ import '../mocks/window.mock';
 import { v4 } from 'uuid';
 
 import { DEFAULT_SESSION_STATE_KEY } from '../../src/common/constants';
+import {
+  CHAT_HISTORY_TYPE,
+  HistoryItem,
+} from '../../src/common/data_structures/history';
 import { protoTimestamp } from '../../src/common/helpers';
-import { CHAT_HISTORY_TYPE, HistoryItem } from '../../src/components/history';
 import { ConnectionService } from '../../src/services/connection.service';
 import { SessionStateService } from '../../src/services/session_state.service';
 import { StateSerializationService } from '../../src/services/wrappers/state_serialization.service';

packages/web-core/package.json

@@ -34,7 +34,8 @@
     "prettier:format": "prettier --write \"./**/*.{js,jsx,ts,tsx}\" --ignore-path ../../.eslintignore --config ../../.prettierrc.json",
     "test": "jest --no-cache --reporters=default",
     "test:coverage": "jest --coverage",
-    "minify": "rollup --config rollup.config.mjs"
+    "minify": "rollup --config rollup.config.mjs",
+    "circular-deps": "madge --circular --extensions ts,tsx src"
   },
   "devDependencies": {
     "@types/defer-promise": "^1.0.0",

packages/web-core/src/clients/inworld.client.ts

@@ -3,18 +3,20 @@ import {
   Awaitable,
   Client,
   ClientConfiguration,
-  Extension,
   GenerateSessionTokenFn,
-  HistoryChangedProps,
   OnPhomeneFn,
   User,
 } from '../common/data_structures';
+import { Extension } from '../common/data_structures/extension';
+import {
+  HistoryChangedProps,
+  HistoryItem,
+} from '../common/data_structures/history';
 import {
   SCENE_HAS_INVALID_FORMAT,
   STOP_DURATION_NATURAL_NUMBER,
   STOP_TICKS_NATURAL_NUMBER,
 } from '../common/errors';
-import { HistoryItem } from '../components/history';
 import { GrpcAudioPlayback } from '../components/sound/grpc_audio.playback';
 import { GrpcAudioRecorder } from '../components/sound/grpc_audio.recorder';
 import { GrpcWebRtcLoopbackBiDiSession } from '../components/sound/grpc_web_rtc_loopback_bidi.session';