From 763d7d76499aca75708db7d558a32af1c4909b21 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Mon, 13 Jan 2025 16:18:52 -0300 Subject: [PATCH 1/4] chore(GatewaysDataSource): more network logs --- src/data/gateways-data-source.ts | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/data/gateways-data-source.ts b/src/data/gateways-data-source.ts index ee297ae9..b085a541 100644 --- a/src/data/gateways-data-source.ts +++ b/src/data/gateways-data-source.ts @@ -104,6 +104,42 @@ export class GatewaysDataSource implements ContiguousDataSource { timeout: this.requestTimeoutMs, }); + gatewayAxios.interceptors.request.use((config) => { + this.log.debug('Axios request initiated', { + url: config.url, + method: config.method, + headers: config.headers, + params: config.params, + timeout: config.timeout, + }); + return config; + }); + + gatewayAxios.interceptors.response.use( + (response) => { + this.log.debug('Axios response received', { + url: response.config.url, + status: response.status, + headers: response.headers, + }); + return response; + }, + (error) => { + if (error.response) { + this.log.error('Axios response error', { + url: error.response.config.url, + status: error.response.status, + headers: error.response.headers, + }); + } else { + this.log.error('Axios network error', { + message: error.message, + }); + } + return Promise.reject(error); + }, + ); + this.log.debug('Attempting to fetch contiguous data from gateway', { id, gatewayUrl, From 2d369c94604005cc2bd7b8ff1fffeea21f4de0d0 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Mon, 13 Jan 2025 16:21:34 -0300 Subject: [PATCH 2/4] feat(GatewaysDataSource): add TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS --- docker-compose.yaml | 1 + docs/envs.md | 1 + src/config.ts | 5 +++++ src/data/gateways-data-source.test.ts | 16 +++++++++++----- src/data/gateways-data-source.ts | 5 ++--- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index f340dec6..fad1e2c8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -47,6 +47,7 @@ services: - TRUSTED_NODE_URL=${TRUSTED_NODE_URL:-} - TRUSTED_GATEWAY_URL=https://${TRUSTED_GATEWAY_HOST:-arweave.net} - TRUSTED_GATEWAYS_URLS=${TRUSTED_GATEWAYS_URLS:-} + - TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS=${TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS:-} - START_HEIGHT=${START_HEIGHT:-} - STOP_HEIGHT=${STOP_HEIGHT:-} - SKIP_CACHE=${SKIP_CACHE:-} diff --git a/docs/envs.md b/docs/envs.md index c7e927d9..a75686ed 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -9,6 +9,7 @@ This document describes the environment variables that can be used to configure | TRUSTED_NODE_URL | String | "https://arweave.net" | Arweave node to use for fetching data | | TRUSTED_GATEWAY_URL | String | "https://arweave.net" | Arweave node to use for proxying requests | | TRUSTED_GATEWAYS_URLS | String | TRUSTED_GATEWAY_URL | A JSON map of gateways and priority | +| TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS | String | 10000 | Request timeout in milliseconds for trusted gateways | | TRUSTED_ARNS_GATEWAY_URL | String | "https://__NAME__.arweave.dev" | ArNS gateway | | INSTANCE_ID | String | "" | Adds an "INSTANCE_ID" field to output logs | | LOG_FORMAT | String | "simple" | Sets the format of output logs, accepts "simple" and "json" | diff --git a/src/config.ts b/src/config.ts index 5d310cb7..b2d627c1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -87,6 +87,11 @@ Object.entries(TRUSTED_GATEWAYS_URLS).forEach(([url, weight]) => { } }); +export const TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS = +env.varOrDefault( + 'TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS', + '10000', +); + // Trusted chunk POST URLs (for posting chunks received at /chunk) export const CHUNK_POST_URLS = env .varOrDefault('CHUNK_POST_URLS', `${TRUSTED_NODE_URL}/chunk`) diff --git a/src/data/gateways-data-source.test.ts b/src/data/gateways-data-source.test.ts index bd0ed544..261741be 100644 --- a/src/data/gateways-data-source.test.ts +++ b/src/data/gateways-data-source.test.ts @@ -25,6 +25,14 @@ import * as metrics from '../metrics.js'; import { TestDestroyedReadable, axiosStreamData } from './test-utils.js'; import { Readable } from 'node:stream'; +const axiosMockCommonParams = (config: any) => ({ + interceptors: { + request: { use: () => {} }, // eslint-disable-line @typescript-eslint/no-empty-function + response: { use: () => {} }, // eslint-disable-line @typescript-eslint/no-empty-function + }, + defaults: config, +}); + let log: winston.Logger; let dataSource: GatewaysDataSource; let mockedAxiosInstance: any; @@ -45,9 +53,7 @@ beforeEach(async () => { 'X-AR-IO-Origin': 'node-url', }, }), - defaults: { - baseURL: 'https://gateway.domain', - }, + ...axiosMockCommonParams({ baseURL: 'https://gateway.domain' }), }; mock.method(axios, 'create', () => mockedAxiosInstance); @@ -129,7 +135,7 @@ describe('GatewayDataSource', () => { }, }; }, - defaults: config, + ...axiosMockCommonParams(config), })); await dataSource.getData({ id: 'test-id' }); @@ -167,7 +173,7 @@ describe('GatewayDataSource', () => { }, }; }, - defaults: config, + ...axiosMockCommonParams(config), })); await dataSource.getData({ id: 'test-id' }); diff --git a/src/data/gateways-data-source.ts b/src/data/gateways-data-source.ts index b085a541..f7bdfe14 100644 --- a/src/data/gateways-data-source.ts +++ b/src/data/gateways-data-source.ts @@ -29,8 +29,7 @@ import { RequestAttributes, } from '../types.js'; import * as metrics from '../metrics.js'; - -const DEFAULT_REQUEST_TIMEOUT_MS = 10000; +import * as config from '../config.js'; export class GatewaysDataSource implements ContiguousDataSource { private log: winston.Logger; @@ -40,7 +39,7 @@ export class GatewaysDataSource implements ContiguousDataSource { constructor({ log, trustedGatewaysUrls, - requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS, + requestTimeoutMs = config.TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS, }: { log: winston.Logger; trustedGatewaysUrls: Record; From 14e70be067fe7b46c46d23abae4a37075a03e9ff Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Mon, 13 Jan 2025 16:43:04 -0300 Subject: [PATCH 3/4] feat(BundleRepairWorker): add BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS and BUNDLE_REPAIR_RETRY_BATCH_SIZE --- docker-compose.yaml | 2 ++ docs/envs.md | 6 ++++-- src/config.ts | 10 ++++++++++ src/workers/bundle-repair-worker.ts | 7 +++---- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index fad1e2c8..dddf80d6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -118,6 +118,8 @@ services: - FS_CLEANUP_WORKER_BATCH_SIZE=${FS_CLEANUP_WORKER_BATCH_SIZE:-} - FS_CLEANUP_WORKER_BATCH_PAUSE_DURATION=${FS_CLEANUP_WORKER_BATCH_PAUSE_DURATION:-} - FS_CLEANUP_WORKER_RESTART_PAUSE_DURATION=${FS_CLEANUP_WORKER_RESTART_PAUSE_DURATION:-} + - BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS=${BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS:-} + - BUNDLE_REPAIR_RETRY_BATCH_SIZE=${BUNDLE_REPAIR_RETRY_BATCH_SIZE:-} networks: - ar-io-network depends_on: diff --git a/docs/envs.md b/docs/envs.md index a75686ed..ed51cdc3 100644 --- a/docs/envs.md +++ b/docs/envs.md @@ -9,7 +9,7 @@ This document describes the environment variables that can be used to configure | TRUSTED_NODE_URL | String | "https://arweave.net" | Arweave node to use for fetching data | | TRUSTED_GATEWAY_URL | String | "https://arweave.net" | Arweave node to use for proxying requests | | TRUSTED_GATEWAYS_URLS | String | TRUSTED_GATEWAY_URL | A JSON map of gateways and priority | -| TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS | String | 10000 | Request timeout in milliseconds for trusted gateways | +| TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS | String | "10000" | Request timeout in milliseconds for trusted gateways | | TRUSTED_ARNS_GATEWAY_URL | String | "https://__NAME__.arweave.dev" | ArNS gateway | | INSTANCE_ID | String | "" | Adds an "INSTANCE_ID" field to output logs | | LOG_FORMAT | String | "simple" | Sets the format of output logs, accepts "simple" and "json" | @@ -75,4 +75,6 @@ This document describes the environment variables that can be used to configure | AWS_ENDPOINT | String | undefined | Custom endpoint for AWS services | | AWS_S3_CONTIGUOUS_DATA_BUCKET | String | undefined | AWS S3 bucket name used for storing data | | AWS_S3_CONTIGUOUS_DATA_PREFIX | String | undefined | Prefix for the S3 bucket to organize data | -| CHUNK_POST_MIN_SUCCESS_COUNT | String | "3" | minimum count of 200 responses for of a given chunk to be considered properly seeded | +| CHUNK_POST_MIN_SUCCESS_COUNT | String | "3" | Minimum count of 200 responses for of a given chunk to be considered properly seeded | +| BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS | String | "300" | Interval in seconds for retrying bundles | +| BUNDLE_REPAIR_RETRY_BATCH_SIZE | String | "1000" | Batch size for retrying bundles | diff --git a/src/config.ts b/src/config.ts index b2d627c1..3ac9882d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -257,6 +257,16 @@ export const MAX_FLUSH_INTERVAL_SECONDS = +env.varOrDefault( '600', ); +export const BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS = +env.varOrDefault( + 'BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS', + '300', // 5 minutes +); + +export const BUNDLE_REPAIR_RETRY_BATCH_SIZE = +env.varOrDefault( + 'BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS', + '1000', +); + // // File system cleanup // diff --git a/src/workers/bundle-repair-worker.ts b/src/workers/bundle-repair-worker.ts index 9f46d878..a6c905b1 100644 --- a/src/workers/bundle-repair-worker.ts +++ b/src/workers/bundle-repair-worker.ts @@ -16,15 +16,14 @@ * along with this program. If not, see . */ import * as winston from 'winston'; +import * as config from '../config.js'; import { BundleIndex } from '../types.js'; import { TransactionFetcher } from './transaction-fetcher.js'; -const DEFAULT_RETRY_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const DEFAULT_UPDATE_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const DEFAULT_BUNDLE_BACKFILL_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes const DEFAULT_FILTER_REPOCESS_INTERVAL_MS = 15 * 60 * 1000; // 15 minutes -const DEFAULT_BUNDLES_TO_RETRY = 1000; export class BundleRepairWorker { // Dependencies @@ -66,7 +65,7 @@ export class BundleRepairWorker { async start(): Promise { const defaultInterval = setInterval( this.retryBundles.bind(this), - DEFAULT_RETRY_INTERVAL_MS, + config.BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS * 1000, ); this.intervalIds.push(defaultInterval); const defaultUpdateInterval = setInterval( @@ -104,7 +103,7 @@ export class BundleRepairWorker { async retryBundles() { try { const bundleIds = await this.bundleIndex.getFailedBundleIds( - DEFAULT_BUNDLES_TO_RETRY, + config.BUNDLE_REPAIR_RETRY_BATCH_SIZE, ); for (const bundleId of bundleIds) { this.log.info('Retrying failed bundle', { bundleId }); From a7e552bfdc9751212a4db83fb92455ed5f7e150c Mon Sep 17 00:00:00 2001 From: David Whittington Date: Wed, 15 Jan 2025 10:17:45 -0600 Subject: [PATCH 4/4] fix(config): correct BUNDLE_REPAIR_RETRY_BATCH_SIZE typo Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 78cd4bd0..f20b8568 100644 --- a/src/config.ts +++ b/src/config.ts @@ -263,7 +263,7 @@ export const BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS = +env.varOrDefault( ); export const BUNDLE_REPAIR_RETRY_BATCH_SIZE = +env.varOrDefault( - 'BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS', + 'BUNDLE_REPAIR_RETRY_BATCH_SIZE', '1000', );