Haff/ep dedupe

packages/event-producer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tidal-music/event-producer",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "type": "module",
   "files": [
     "dist"

packages/event-producer/src/submit/submit.test.ts

@@ -279,4 +279,24 @@ describe.sequential('submit', () => {
     expect(monitor.registerDroppedEvent).not.toHaveBeenCalled();
     expect(queue.removeEvents).toHaveBeenCalledWith([]);
   });
+
+  it('error response with AWS.SimpleQueueService.BatchEntryIdsNotDistinct removes duplicates', async () => {
+    vi.mocked(queue).getEventBatch.mockReturnValue([epEvent1, epEvent1]);
+    vi.mocked(queue).getEvents.mockReturnValue([epEvent1, epEvent1]);
+    vi.stubGlobal(
+      'fetch',
+      vi.fn().mockResolvedValueOnce({
+        ok: false,
+        text: vi
+          .fn()
+          .mockResolvedValueOnce(
+            '<?xml version="1.0"?><ErrorResponse xmlns="http://queue.amazonaws.com/doc/2012-11-05/"><Error><Type>Sender</Type><Code>AWS.SimpleQueueService.BatchEntryIdsNotDistinct</Code><Message>Id a6f4964d-63da-4d66-86bf-e9155b4bb499 repeated.</Message><Detail/></Error><RequestId>635d186a-54a0-52e5-b8c0-4601e8f85440</RequestId></ErrorResponse>',
+          ),
+      }),
+    );
+
+    await submitEvents({ config });
+
+    expect(queue.setEvents).toHaveBeenCalledWith([epEvent1]);
+  });
 });

packages/event-producer/src/submit/submit.ts

@@ -4,6 +4,7 @@ import type { Config } from '../config';
 import * as monitor from '../monitor';
 import { isOutage, setOutage } from '../outage';
 import * as queue from '../queue';
+import type { EPEvent } from '../types';
 import { eventsToSqsRequestParameters } from '../utils/sqsParamsConverter';
 
 /**
@@ -90,6 +91,31 @@ export const submitEvents = async ({
     const respStr = await res.text();
     console.error('Error sending event batch:', respStr);
     setOutage(true);
+
+    const xml = new window.DOMParser().parseFromString(respStr, 'text/xml');
+    if (
+      xml.querySelector('ErrorResponse Error Type')?.textContent === 'Sender'
+    ) {
+      // If the error is due to duplicate event ids, we dedupe the queue for next run.
+      if (
+        xml.querySelector('ErrorResponse Error Code')?.textContent ===
+        'AWS.SimpleQueueService.BatchEntryIdsNotDistinct'
+      ) {
+        const currentEvents = queue.getEvents();
+        const eventData: Record<string, EPEvent> = {};
+        const uniqueIds = new Set(
+          currentEvents.map(event => {
+            eventData[event.id] = event;
+            return event.id;
+          }),
+        );
+        const dedupedEvents = Array.from(uniqueIds).map(
+          id => eventData[id],
+        ) as Array<EPEvent>;
+
+        queue.setEvents(dedupedEvents);
+      }
+    }
   }
   return Promise.resolve();
 };