diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f9d75..eac219f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,28 @@ jobs: with: path: cosmos + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + + - name: Pull and run Azure Cosmos DB Emulator + run: | + docker pull mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator + docker run -d --name=cosmos-emulator -p 8081:8081 -p 10251:10251 -p 10252:10252 -p 10253:10253 -p 10254:10254 \ + -e AZURE_COSMOS_EMULATOR_PARTITION_COUNT=10 \ + -e AZURE_COSMOS_EMULATOR_ENABLE_DATA_PERSISTENCE=false \ + mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator + + - name: Wait for Cosmos DB Emulator to be ready + run: | + timeout 300 bash -c 'until curl -ks https://localhost:8081/_explorer/emulator.pem > /dev/null; do sleep 5; done' + echo "Cosmos DB Emulator is ready" + + - name: Download Cosmos DB Emulator certificate + run: | + curl -k https://localhost:8081/_explorer/emulator.pem > emulatorcert.crt + sudo cp emulatorcert.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + - name: Build connector run: | cd cosmos @@ -33,9 +55,9 @@ jobs: - name: Run tests env: - AZURE_COSMOS_KEY: ${{ secrets.AZURE_COSMOS_KEY }} - AZURE_COSMOS_ENDPOINT: ${{ secrets.AZURE_COSMOS_ENDPOINT }} - AZURE_COSMOS_DB_NAME: ${{ secrets.AZURE_COSMOS_DB_NAME }} + AZURE_COSMOS_KEY: C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== + AZURE_COSMOS_ENDPOINT: https://localhost:8081 + AZURE_COSMOS_DB_NAME: TestNobelLaureates run: | cd cosmos - npm run ndc-test + npm run ndc-test -- --setup-emulator-data diff --git a/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/expected.json b/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/expected.json index f31ac17..820d474 100644 --- a/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/expected.json +++ b/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/expected.json @@ -1,134 +1,132 @@ [ - { - "rows": [ - { - "laureates": [ - { - "firstName": "James P.", - "id": "958" - }, - { - "firstName": "Tasuku", - "id": "959" - } - ] - }, - { - "laureates": [ - { - "firstName": "Oliver", - "id": "935" - }, - { - "firstName": "Bengt", - "id": "936" - } - ] - }, - { - "laureates": [ - { - "firstName": "David J.", - "id": "928" - }, - { - "firstName": "F. Duncan M.", - "id": "929" - }, - { - "firstName": "J. Michael", - "id": "930" - } - ] - }, - { - "laureates": [ - { - "firstName": "Arthur", - "id": "960" - }, - { - "firstName": "Gérard", - "id": "961" - }, - { - "firstName": "Donna", - "id": "962" - } - ] - }, - { - "laureates": [ - { - "firstName": "Katalin", - "id": "1024" - }, - { - "firstName": "Drew", - "id": "1025" - } - ] - }, - { - "laureates": [ - { - "firstName": "Abiy", - "id": "981" - } - ] - }, - { - "laureates": [ - { - "firstName": "Serge", - "id": "876" - }, - { - "firstName": "David J.", - "id": "877" - } - ] - }, - { - "laureates": [ - { - "firstName": "Intergovernmental Panel on Climate Change", - "id": "818" - }, - { - "firstName": "Al", - "id": "819" - } - ] - }, - { - "laureates": [ - { - "firstName": "François", - "id": "887" - }, - { - "firstName": "Peter", - "id": "888" - } - ] - }, - { - "laureates": [ - { - "firstName": "Bruce A.", - "id": "861" - }, - { - "firstName": "Jules A.", - "id": "862" - }, - { - "firstName": "Ralph M.", - "id": "863" - } - ] - } + { + "rows": [ + { + "year": 2023, + "laureates": [ + { + "firstName": "Katalin", + "id": "1024" + }, + { + "firstName": "Drew", + "id": "1025" + } ] - } + }, + { + "year": 2023, + "laureates": [ + { + "firstName": "Pierre", + "id": "1026" + }, + { + "firstName": "Ferenc", + "id": "1027" + }, + { + "firstName": "Anne", + "id": "1028" + } + ] + }, + { + "year": 2023, + "laureates": [ + { + "firstName": "Narges", + "id": "1033" + } + ] + }, + { + "year": 2023, + "laureates": [ + { + "firstName": "Jon", + "id": "1032" + } + ] + }, + { + "year": 2023, + "laureates": [ + { + "firstName": "Claudia", + "id": "1034" + } + ] + }, + { + "year": 2023, + "laureates": [ + { + "firstName": "Moungi", + "id": "1029" + }, + { + "firstName": "Louis", + "id": "1030" + }, + { + "firstName": "Aleksey", + "id": "1031" + } + ] + }, + { + "year": 2022, + "laureates": [ + { + "firstName": "Svante", + "id": "1011" + } + ] + }, + { + "year": 2022, + "laureates": [ + { + "firstName": "Alain", + "id": "1012" + }, + { + "firstName": "John ", + "id": "1013" + }, + { + "firstName": "Anton", + "id": "1014" + } + ] + }, + { + "year": 2022, + "laureates": [ + { + "firstName": "Ales", + "id": "1018" + }, + { + "firstName": "Memorial", + "id": "1019" + }, + { + "firstName": "Center for Civil Liberties", + "id": "1020" + } + ] + }, + { + "year": 2022, + "laureates": [ + { + "firstName": "Annie", + "id": "1017" + } + ] + } + ] + } ] diff --git a/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/request.json b/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/request.json index d21437e..1499af7 100644 --- a/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/request.json +++ b/ndc-test-snapshots/query/nobel_laureates_nested_array_field_selection/request.json @@ -2,26 +2,30 @@ "collection": "TestNobelLaureates", "query": { "fields": { + "year": { + "type": "column", + "column": "year" + }, "laureates": { "type": "column", "column": "laureates", "fields": { - "type": "array", + "type": "array", + "fields": { + "type": "object", "fields": { - "type": "object", - "fields": { - "firstName": { - "type": "column", - "column": "firstname" - }, - "id": { - "type": "column", - "column": "id" - } - } + "firstName": { + "type": "column", + "column": "firstname" + }, + "id": { + "type": "column", + "column": "id" + } } + } } - } + } }, "limit": 10, "order_by": { @@ -30,7 +34,7 @@ "order_direction": "desc", "target": { "type": "column", - "name": "id", + "name": "year", "path": [] } } diff --git a/script/test-setup.js b/script/test-setup.js index 1ff4076..89b2e52 100644 --- a/script/test-setup.js +++ b/script/test-setup.js @@ -1,4 +1,5 @@ import { exec, execSync } from 'child_process'; +import { CosmosClient } from '@azure/cosmos'; import net from 'net'; import path from 'path'; import fs from 'fs'; @@ -7,6 +8,133 @@ import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +// Azure Cosmos DB configuration +const cosmosConfig = { + endpoint: process.env.AZURE_COSMOS_ENDPOINT, + key: process.env.AZURE_COSMOS_KEY, + databaseId: process.env.AZURE_COSMOS_DB_NAME, + containerId: 'TestNobelLaureates' // Use the existing container +}; + +// Validate Cosmos DB configuration +Object.entries(cosmosConfig).forEach(([key, value]) => { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Invalid or missing Cosmos DB configuration: ${key}`); + } +}); + +const client = new CosmosClient({ endpoint: cosmosConfig.endpoint, key: cosmosConfig.key }); + +function readTestData() { + const testDataPath = path.join(__dirname, 'data', 'data.json'); + try { + const rawData = fs.readFileSync(testDataPath, 'utf8'); + return JSON.parse(rawData); + } catch (error) { + console.error('Error reading test data:', error); + throw error; + } +} + +async function setupCosmosEmulatorDB() { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0 + + console.log('Setting up Azure Cosmos DB...'); + console.log(`Database ID: ${cosmosConfig.databaseId}`); + console.log(`Container ID: ${cosmosConfig.containerId}`); + + try { + const { database } = await client.databases.createIfNotExists({ id: cosmosConfig.databaseId }); + const { container } = await database.containers.createIfNotExists( + { + id: cosmosConfig.containerId, + partitionKey: { paths: ['/year'] }, + indexingPolicy: { + "indexingMode": "consistent", + "automatic": true, + "includedPaths": [ + { + "path": "/*" + } + ], + "excludedPaths": [ + { + "path": "/\"_etag\"/?" + } + ], + "compositeIndexes": [ + [ + { + "path": "/overallMotivation", + "order": "descending" + }, + { + "path": "/prize_id", + "order": "descending" + } + ], + [ + { + "path": "/overallMotivation", + "order": "ascending" + }, + { + "path": "/prize_id", + "order": "descending" + } + ], + [ + { + "path": "/year", + "order": "ascending" + }, + { + "path": "/prize_id", + "order": "ascending" + } + ], + [ + { + "path": "/year", + "order": "ascending" + }, + { + "path": "/prize_id", + "order": "descending" + } + ] + ] + } + }); + + console.log('Deleting existing data...'); + const { resources: items } = await container.items.readAll().fetchAll(); + if (items.length > 0) { + for (const item of items) { + await container.item(item.id, item.year).delete(); + } + console.log(`${items.length} items deleted.`); + } + + console.log('Inserting new test data...'); + const newItems = readTestData(); + for (const item of newItems) { + await container.items.create(item); + } + console.log(`${newItems.length} items inserted.`); + console.log('Data setup completed.'); + } catch (error) { + console.error('Error setting up Cosmos DB:', error); + if (error.code) { + console.error(`Error code: ${error.code}`); + } + if (error.body) { + console.error('Error details:', error.body); + } + throw error; + } +} + function findVacantPort(startPort = 8000) { return new Promise((resolve, reject) => { const server = net.createServer(); @@ -24,15 +152,21 @@ function findVacantPort(startPort = 8000) { }); } -async function runTests() { +async function runTests(setupData) { const projectRoot = path.resolve(__dirname, '..'); const logFile = path.join(projectRoot, 'server-output.log'); let serverProcess = null; try { - console.log('Generating project configuration...'); + if (setupData) { + await setupCosmosEmulatorDB(); + } else { + console.log('Skipping Cosmos DB setup as --setup-emulator-data flag is not set.'); + } + + console.log('Running CLI update...'); execSync('npm run cli update', { stdio: 'inherit', cwd: projectRoot }); - console.log('Configuration generated successfully.'); + console.log('CLI update completed.'); const port = await findVacantPort(); console.log(`Found vacant port: ${port}`); @@ -66,6 +200,7 @@ async function runTests() { } catch (readError) { console.error('Failed to read server output log:', readError); } + throw error; // Re-throw the error to be caught in the main function } finally { // Terminate the server process if (serverProcess) { @@ -84,4 +219,17 @@ async function runTests() { } } -runTests().then(() => process.exit(0)).catch(() => process.exit(1)); +async function main() { + try { + const args = process.argv.slice(2); + const setupData = args.includes('--setup-emulator-data'); + + await runTests(setupData); + process.exit(0); // Successful execution + } catch (error) { + console.error('An error occurred in the main function:', error); + process.exit(1); // Exit with error status + } +} + +main();