Fix #19 add rate limiter - github action npm audit

.github/workflows/main.yml

@@ -58,6 +58,13 @@ jobs:
         github-token: ${{ secrets.GITHUB_TOKEN }}
         lcov-file: ./coverage/lcov.info
 
+    - name: Npm audit
+      uses: oke-py/[email protected]
+      with:
+        audit_level: moderate
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        dedupe_issues: true
+
     - name: Publish NpmJS package
       if: github.ref == 'refs/heads/npmjs'
       run: |

README.md

@@ -50,6 +50,48 @@ You could avoid using environment variable by using constructor options:
 var client = new EtsyClient({apiKey:'mSecretHere'});
 ```
 
+## Advanced usage
+
+
+### Etsy client options
+This section describes EtsyClient available options.
+
+Note about options precedence: first take option value from constructor if any, or
+else try to retrieve related environment variable, or else apply default value.
+
+- `apiUrl` : Etsy api endpoint - **required** (or env.`ETSY_API_ENDPOINT`) default value is `https://openapi.etsy.com/v2`.
+- `apiKey` : Etsy api key - **required** (or env.`ETSY_API_KEY`) without default value. Ask one from [Etsy portal](https://www.etsy.com/developers/documentation/getting_started/register)
+- `shop`   : Etsy shop name - *optional* (or env.`ETSY_SHOP`) without default value.
+- `lang`   : Etsy language - *optional* (or env.`ETSY_LANG`) without default value. Example: `fr`
+- `etsyRateWindowSizeMs` : Rate limit windows size in milliseconds - *optional* (or env.`ETSY_RATE_WINDOWS_SIZE_MS`) with default value: `1000`
+- `etsyRateMaxQueries`   : Rate limit max query per windows size - *optional* (or env.`ETSY_RATE_MAX_QUERIES`) without default value
+- `etsyRateWait`         : On limit reached, should wait for next slot (instead of throwing error) - *optional* (or env.`ETSY_RATE_WAIT`) with default value: `true`
+- `dryMode`              : print call instead of making real etsy call - *optional* (or env.`ETSY_DRY_MODE`) with default value: `false`
+
+Note about rate limit options:
+
+Rate limit is enabled if and only if `etsyRateWindowSizeMs` and `etsyRateMaxQueries` are both set.
+
+This will configure rate limit on etsy call : max `etsyRateMaxQueries` per `etsyRateWindowSizeMs`ms.
+
+On limit reached, if `etsyRateWait`, then wait, else throw an error immediately. 
+
+For more details, cf. [node-rate-limiter](https://github.com/jhurliman/node-rate-limiter)
+
+### Rate limit
+According to [their documentation](https://www.etsy.com/developers/documentation/getting_started/api_basics#section_rate_limiting),
+Etsy restricts number of call to 10 per second (and 10k per day).
+
+In order to never reach this (second windows) rate limit, node-etsy-client rely on [node-rate-limiter](https://github.com/jhurliman/node-rate-limiter) 
+and offer an option to rate limit client calls.
+
+To apply rate limit of 10 query per seconds (with wait on unavailable slot),
+add `etsyRateMaxQueries` option:
+
+```
+var client = new EtsyClient({apiKey:'mSecretHere', etsyRateMaxQueries:10});
+```
+
 ## How to contribute
 You're not a dev ? just submit an issue (bug, improvements, questions). Or else:
 * Clone
@@ -72,6 +114,9 @@ npm run test
 
 | badge  | name   | description  |
 |--------|-------|:--------|
-| [![Build Status](https://travis-ci.com/creharmony/node-etsy-client.svg?branch=main)](https://travis-ci.com/creharmony/node-etsy-client) |[Travis-ci](https://travis-ci.com/creharmony/node-etsy-client)|Continuous tests.
+| ![CI/CD](https://github.com/creharmony/node-etsy-client/workflows/etsy_client_ci/badge.svg) |Github actions|Continuous tests.
 |  |[Houndci](https://houndci.com/)|JavaScript  automated review (configured by `.hound.yml`)|
 | [![Automated Release Notes by gren](https://img.shields.io/badge/%F0%9F%A4%96-release%20notes-00B2EE.svg)](https://github-tools.github.io/github-release-notes/)|[gren](https://github.com/github-tools/github-release-notes)|[Release notes](https://github.com/creharmony/node-etsy-client/releases) automation|
+<!-- travis disabled
+| [![Build Status](https://travis-ci.com/creharmony/node-etsy-client.svg?branch=main)](https://travis-ci.com/creharmony/node-etsy-client) |[Travis-ci](https://travis-ci.com/creharmony/node-etsy-client)|Continuous tests.
+-->

package.json

@@ -35,6 +35,7 @@
   },
   "dependencies": {
     "chai": "^4.2.0",
+    "limiter": "^1.1.5",
     "mocha": "^8.2.1",
     "node-fetch": "^2.6.1",
     "nyc": "^15.1.0",

src/EtsyClient.js

@@ -1,5 +1,9 @@
 const queryString = require('query-string');
 const fetch = require("node-fetch");
+const RateLimiter = require('limiter').RateLimiter;
+
+const DEFAULT_DRY_MODE = false;// DEBUG // if true, then print fetch onto console instead of calling etsy
+const DEFAULT_RATE_WAIT = true;// rate limite will wait next available query instead of reporting an error immediately
 
 /**
  * this utility class implement ETSY (some) methods documented here:
@@ -13,93 +17,143 @@ class EtsyClient {
     if (!options) {
       options = {};
     }
-    this.apiUrl = options.apiUrl || process.env.ETSY_API_ENDPOINT || 'https://openapi.etsy.com/v2'
-    this.apiKey = options.apiKey || process.env.ETSY_API_KEY;
-    this.shop   = options.shop   || process.env.ETSY_SHOP;
-    this.lang   = options.lang   || process.env.ETSY_LANG;
+    this.apiUrl = "apiUrl" in options ? options.apiUrl : (process.env.ETSY_API_ENDPOINT || 'https://openapi.etsy.com/v2');
+    this._assumeApiUrl();
+    this.apiKey = "apiKey" in options ? options.apiKey : process.env.ETSY_API_KEY;
     this._assumeApiKey();
+    this.shop   = "shop" in options ? options.shop : process.env.ETSY_SHOP;
+    this.lang   = "lang" in options ? options.lang : process.env.ETSY_LANG;
+    // configure rate limit on etsy call : max <etsyRateMaxQueries> per <etsyRateWindowSizeMs> ms
+    // on limit reach, wait if <etsyRateWait>, else throw an error immediately. cf. https://github.com/jhurliman/node-rate-limiter
+    // Etsy rate limit doc (10/sec) : https://www.etsy.com/developers/documentation/getting_started/api_basics#section_rate_limiting
+    this.etsyRateWindowSizeMs = "etsyRateWindowSizeMs" in options ? options.etsyRateWindowSizeMs : (process.env.ETSY_RATE_WINDOWS_SIZE_MS  || 1000);
+    this.etsyRateMaxQueries   = "etsyRateMaxQueries" in options ? options.etsyRateMaxQueries : (process.env.ETSY_RATE_MAX_QUERIES || null);
+    this.etsyRateWait         = "etsyRateWait" in options ? options.etsyRateWait : ("true" === process.env.ETSY_RATE_WAIT || true);
+    this.dryMode              = "dryMode" in options ? options.dryMode : ("true" === process.env.ETSY_DRY_MODE || DEFAULT_DRY_MODE);
+    this.initRateLimiter();
+    // DEBUG // console.debug(`EtsyClient - apiUrl:${this.apiUrl} - dryMode:${this.dryMode} ${this.limiterDesc}`);
+  }
+
+  initRateLimiter() {
+    this.limiter = this.etsyRateWindowSizeMs === null || this.etsyRateMaxQueries === null ? null :
+      new RateLimiter(this.etsyRateMaxQueries, this.etsyRateWindowSizeMs, (!this.etsyRateWait));
+    this.limiterDesc = (this.limiter === null) ? "" : `rate limit:${this.etsyRateMaxQueries} query / ${this.etsyRateWindowSizeMs}ms` +
+      !this.etsyRateWait ? "throwing error on limit reached":"";
+  }
+
+  isRateLimitEnabled() {
+    return this.limiter !== null;
   }
 
   // https://www.etsy.com/developers/documentation/reference/shop#method_findallshops
   findAllShops(options) {
-    return this.etsyApiFetch(`/shops`, options);
+    return this.limitedEtsyApiFetch(`/shops`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/shop#method_getshop
   getShop(options) {
      this._assumeShop();
-     return this.etsyApiFetch(`/shops/${this.shop}`, options);
+     return this.limitedEtsyApiFetch(`/shops/${this.shop}`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/shopsection#method_findallshopsections
   findAllShopSections(listingId, options) {
      this._assumeShop();
-     return this.etsyApiFetch(`/shops/${this.shop}/sections`, options);
+     return this.limitedEtsyApiFetch(`/shops/${this.shop}/sections`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listing#method_findallshoplistingsactive
   findAllShopListingsActive(options) {
      this._assumeShop();
-     return this.etsyApiFetch(`/shops/${this.shop}/listings/active`, options);
+     return this.limitedEtsyApiFetch(`/shops/${this.shop}/listings/active`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listing#method_getlisting
   getListing(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listingvariationimage#method_getvariationimages
   getVariationImages(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}/variation-images`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/variation-images`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listingimage#method_findalllistingimages
   findAllListingImages(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}/images`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/images`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listinginventory
   getInventory(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}/inventory`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/inventory`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/propertyvalue#method_getattribute
   getAttributes(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}/attributes`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/attributes`, options);
   }
 
   // https://www.etsy.com/developers/documentation/reference/listingproduct#method_getproduct
   getProduct(listingId, productId, options) {
      this._assumeField('listingId', listingId);
      this._assumeField('productId', productId);
-     return this.etsyApiFetch(`/listings/${listingId}/products/${productId}`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/products/${productId}`, options);
   }
 
-
   // https://www.etsy.com/developers/documentation/reference/shippinginfo#method_findalllistingshippingprofileentries
   findAllListingShippingProfileEntries(listingId, options) {
      this._assumeField('listingId', listingId);
-     return this.etsyApiFetch(`/listings/${listingId}/shipping/info`, options);
+     return this.limitedEtsyApiFetch(`/listings/${listingId}/shipping/info`, options);
+  }
+
+  limitedEtsyApiFetch(endpoint, options) {
+    var client = this;
+    if (!client.isRateLimitEnabled()) {
+      return client.safeEtsyApiFetch(endpoint, options);
+    } else {
+      return new Promise((resolve, reject) => {
+        client.asyncRemoveTokens(1)
+          .then((remainingRequests)=> {
+            client.safeEtsyApiFetch(endpoint, options)
+                .then(resolve).catch(reject);
+          })
+          .catch(reject)
+      });
+    }
   }
 
-  etsyApiFetch(endpoint, options) {
+  asyncRemoveTokens(count) {// hurliman/node-rate-limiter/issues/63 by sunknudsen
+    var client = this;
+    return new Promise((resolve, reject) => {
+      client.limiter.removeTokens(count, (error, remainingRequests) => {
+        if (remainingRequests < 0 && !client.etsyRateWait) {
+          reject("rate limit reached");
+        }
+        if (error) return reject(error)
+        resolve(remainingRequests)
+      })
+    })
+  }
+
+  safeEtsyApiFetch(endpoint, options) {
      this._assumeField('endpoint', endpoint);
+     var client = this;
      return new Promise((resolve, reject) => {
-         const getQueryString = queryString.stringify(this.getOptions(options));
-         fetch(`${this.apiUrl}${endpoint}?${getQueryString}`)
+         const getQueryString = queryString.stringify(client.getOptions(options));
+         client.nodeFetch(`${client.apiUrl}${endpoint}?${getQueryString}`)
            .then(response => EtsyClient._response(response, resolve, reject))
            .catch((fetchError) => {
              var secureError = {};
-             this.secureErrorAttribute(secureError, fetchError, "message");
-             this.secureErrorAttribute(secureError, fetchError, "reason");
-             this.secureErrorAttribute(secureError, fetchError, "type");
-             this.secureErrorAttribute(secureError, fetchError, "errno");
-             this.secureErrorAttribute(secureError, fetchError, "code");
+             client.secureErrorAttribute(secureError, fetchError, "message");
+             client.secureErrorAttribute(secureError, fetchError, "reason");
+             client.secureErrorAttribute(secureError, fetchError, "type");
+             client.secureErrorAttribute(secureError, fetchError, "errno");
+             client.secureErrorAttribute(secureError, fetchError, "code");
              reject(secureError);
            });
      });
@@ -125,8 +179,24 @@ class EtsyClient {
     return merged;
   }
 
+  dryFetch(endpoint) {
+    const response = {};
+    response.ok = true;
+    response.text = function(){ return JSON.stringify({endpoint});};
+    console.log(`[dry_fetch] ${endpoint}`);
+    return Promise.resolve(response);
+  }
+
+  nodeFetch(endpoint) {
+    if (this.dryMode) {
+      return this.dryFetch(endpoint);
+    }
+    return fetch(endpoint);
+  }
+
   _assumeShop() { if (!this.shop) { throw "shop is not defined";  } }
-  _assumeApiKey() { if (!this.apiKey) { throw "apiKey is required. You must set ETSY_API_KEY environment variable.";  } }
+  _assumeApiUrl() { if (!this.apiUrl) { throw "apiUrl is required. ie. set ETSY_API_ENDPOINT environment variable.";  } }
+  _assumeApiKey() { if (!this.apiKey) { throw "apiKey is required. ie. set ETSY_API_KEY environment variable.";  } }
   _assumeField(fieldName, fieldValue) { if (!fieldValue) { throw fieldName + " is required";  } }
 
   static _response(response, resolve, reject) {

tests/unauthenticated_client.test.js

@@ -23,6 +23,86 @@ if (!process.env.ETSY_API_KEY) {
         })
     });
 
+    // 11 test cases (>10)
+    async function api_test_cases(etsyExtraOptions) {
+      var etsyOptions = etsyExtraOptions;
+      const apiKey = FAKE_API_KEY;
+      const apiUrl = "https://IAmNotEtsyEndpoint.com";
+      const shop = "MyShop";
+      const lang = "fr";
+      const listingId = "12345665432";
+      const productId = "34555555555";
+
+      etsyOptions.apiKey = apiKey;
+      etsyOptions.apiUrl = apiUrl;
+      etsyOptions.shop = shop;
+      etsyOptions.lang = lang;
+
+      const client = new EtsyClient(etsyOptions);
+      const expectedEndpoint = (path) => `${apiUrl}${path}?api_key=${apiKey}&language=${lang}`;
+
+      const findAllShops = await client.findAllShops().catch(_expectNoError);
+      expect(findAllShops.endpoint).to.be.eql(expectedEndpoint("/shops"));
+
+      const getShop = await client.getShop().catch(_expectNoError);
+      expect(getShop.endpoint).to.be.eql(expectedEndpoint(`/shops/${shop}`));
+
+      const findAllShopSections = await client.findAllShopSections().catch(_expectNoError);
+      expect(findAllShopSections.endpoint).to.be.eql(expectedEndpoint(`/shops/${shop}/sections`));
+
+      const findAllShopListingsActive = await client.findAllShopListingsActive().catch(_expectNoError);
+      expect(findAllShopListingsActive.endpoint).to.be.eql(expectedEndpoint(`/shops/${shop}/listings/active`));
+
+      const getListing = await client.getListing(listingId).catch(_expectNoError);
+      expect(getListing.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}`));
+
+      const getVariationImages = await client.getVariationImages(listingId).catch(_expectNoError);
+      expect(getVariationImages.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/variation-images`));
+
+      const findAllListingImages = await client.findAllListingImages(listingId).catch(_expectNoError);
+      expect(findAllListingImages.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/images`));
+
+      const getInventory = await client.getInventory(listingId).catch(_expectNoError);
+      expect(getInventory.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/inventory`));
+
+      const getAttributes = await client.getAttributes(listingId).catch(_expectNoError);
+      expect(getAttributes.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/attributes`));
+
+      const getProduct = await client.getProduct(listingId, productId).catch(_expectNoError);
+      expect(getProduct.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/products/${productId}`));
+
+      if ("etsyRateWait" in etsyOptions && etsyOptions.etsyRateWait === false) {
+              await client.findAllListingShippingProfileEntries(listingId)
+              .catch((rateLimitError) => {
+                expect(rateLimitError).to.be.eql("rate limit reached");
+              })
+      } else {
+          const findAllListingShippingProfileEntries = await client.findAllListingShippingProfileEntries(listingId).catch(_expectNoError);
+          expect(findAllListingShippingProfileEntries.endpoint).to.be.eql(expectedEndpoint(`/listings/${listingId}/shipping/info`));
+      }
+    }
+
+    it("should dry test cases without rate limit", async function() {
+      const start = new Date()
+      await api_test_cases({dryMode:true})
+      const duration = new Date() - start;
+      expect(duration).to.be.lt(500);
+    });
+
+    it("should dry test cases with rate limit of 10 per seconds", async function() {
+      const start = new Date()
+      await api_test_cases({dryMode:true, etsyRateMaxQueries:10})
+      const duration = new Date() - start;
+      expect(duration).to.be.gt(1000);
+    });
+
+    it("should dry test cases with rate limit of 10 per seconds without wait", async function() {
+      await api_test_cases({dryMode:true, etsyRateMaxQueries:10, etsyRateWait:false});
+    });
   });
+}
 
-}
+function _expectNoError(err) {
+  console.trace(err)
+  expect.fail(err);
+}