feat: add preprocessors

__tests__/register/preprocessor.test.js

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with
+ * the License. A copy of the License is located at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+import { expect } from 'chai';
+import StyleDictionary from 'style-dictionary';
+
+describe('register/transformGroup', async () => {
+  let StyleDictionaryExtended;
+  beforeEach(async () => {
+    StyleDictionary.preprocessors = {};
+    StyleDictionaryExtended = new StyleDictionary({});
+    await StyleDictionaryExtended.hasInitialized;
+  });
+
+  it('should support registering preprocessor on StyleDictionary class', () => {
+    StyleDictionary.registerPreprocessor({
+      name: 'example-preprocessor',
+      preprocessor: (dict) => dict,
+    });
+    expect(StyleDictionary.preprocessors['example-preprocessor']).to.not.be.undefined;
+    expect(StyleDictionaryExtended.preprocessors['example-preprocessor']).to.not.be.undefined;
+  });
+
+  it('should support registering preprocessor on StyleDictionary instance, which registers it on the class', () => {
+    StyleDictionaryExtended.registerPreprocessor({
+      name: 'example-preprocessor',
+      preprocessor: (dict) => dict,
+    });
+    expect(StyleDictionary.preprocessors['example-preprocessor']).to.not.be.undefined;
+    expect(StyleDictionaryExtended.preprocessors['example-preprocessor']).to.not.be.undefined;
+  });
+
+  it('should throw if the preprocessor name is not a string', () => {
+    expect(() => {
+      StyleDictionaryExtended.registerPreprocessor({
+        name: true,
+        preprocessor: (dict) => dict,
+      });
+    }).to.throw('Cannot register preprocessor; Preprocessor.name must be a string');
+  });
+
+  it('should throw if the preprocessor is not a function', () => {
+    expect(() => {
+      StyleDictionaryExtended.registerPreprocessor({
+        name: 'example-preprocessor',
+        preprocessor: 'foo',
+      });
+    }).to.throw('Cannot register preprocessor; Preprocessor.preprocessor must be a function');
+  });
+
+  it('should preprocess the dictionary as specified', async () => {
+    StyleDictionary.registerPreprocessor({
+      name: 'strip-descriptions',
+      preprocessor: (dict) => {
+        // recursively traverse token objects and delete description props
+        function removeDescription(slice) {
+          delete slice.description;
+          Object.values(slice).forEach((value) => {
+            if (typeof value === 'object') {
+              removeDescription(value);
+            }
+          });
+          return slice;
+        }
+        return removeDescription(dict);
+      },
+    });
+
+    StyleDictionaryExtended = new StyleDictionary({
+      tokens: {
+        foo: {
+          value: '4px',
+          type: 'dimension',
+          description: 'Foo description',
+        },
+        description: 'My dictionary',
+      },
+    });
+    await StyleDictionaryExtended.hasInitialized;
+    expect(StyleDictionaryExtended.tokens).to.eql({
+      foo: {
+        value: '4px',
+        type: 'dimension',
+      },
+    });
+  });
+
+  it('should support async preprocessors', async () => {
+    StyleDictionary.registerPreprocessor({
+      name: 'strip-descriptions',
+      preprocessor: async (dict) => {
+        // recursively traverse token objects and delete description props
+        async function removeDescription(slice) {
+          // Arbitrary delay, act as though this action is asynchronous and takes some time
+          await new Promise((resolve) => setTimeout(resolve, 100));
+          delete slice.description;
+
+          await Promise.all(
+            Object.values(slice).map((value) => {
+              if (typeof value === 'object') {
+                return removeDescription(value);
+              } else {
+                return Promise.resolve();
+              }
+            }),
+          );
+          return slice;
+        }
+        return removeDescription(dict);
+      },
+    });
+
+    StyleDictionaryExtended = new StyleDictionary({
+      tokens: {
+        foo: {
+          value: '4px',
+          type: 'dimension',
+          description: 'Foo description',
+        },
+        description: 'My dictionary',
+      },
+    });
+    await StyleDictionaryExtended.hasInitialized;
+    expect(StyleDictionaryExtended.tokens).to.eql({
+      foo: {
+        value: '4px',
+        type: 'dimension',
+      },
+    });
+  });
+});

docs/_sidebar.md

@@ -15,6 +15,7 @@
   - [Using the NPM Module](using_the_npm_module.md)
   - [API](api.md)
   - [Parsers](parsers.md)
+  - [Preprocessors](preprocessors.md)
   - [Transforms](transforms.md)
   - [Transform groups](transform_groups.md)
   - [Formats](formats.md)

docs/api.md

@@ -265,6 +265,32 @@ StyleDictionary.registerParser({
 
 ---
 
+### registerPreprocessor
+
+> StyleDictionary.registerPreprocessor({ name, preprocessor }) => [<code>style-dictionary</code>](#module_style-dictionary)
+
+Adds a custom parser to parse style dictionary files
+
+| Param                     | Type                  | Description                                                                                  |
+| ------------------------- | --------------------- | -------------------------------------------------------------------------------------------- |
+| Preprocessor              | <code>Object</code>   |                                                                                              |
+| Preprocessor.name         | <code>String</code>   | Name of the format to be referenced in your config.json                                      |
+| Preprocessor.preprocessor | <code>function</code> | Function to preprocess the dictionary. The function should return a plain Javascript object. |
+
+**Example**
+
+```js
+StyleDictionary.registerPreprocessor({
+  name: 'strip-third-party-meta',
+  preprocessor: (dictionary) => {
+    delete dictionary.thirdPartyMetadata;
+    return dictionary;
+  },
+});
+```
+
+---
+
 ### registerTemplate
 
 > ~~StyleDictionary.registerTemplate(template) ⇒ [<code>style-dictionary</code>](#module_style-dictionary)~~

docs/architecture.md

@@ -20,25 +20,29 @@ If there are [custom parsers](parsers.md) defined, Style Dictionary will run tho
 
 Style Dictionary takes all the files it found and performs a deep merge. This allows you to split your token files in any way you like, without worrying about accidentally overriding groups of tokens. This gives Style Dictionary a single, complete token object to work from.
 
-## 4. Iterate over the platforms
+## 4. Run preprocessors over the dictionary
+
+Allows users to configure [custom preprocessors](preprocessors.md), to process the merged dictionary as a whole, rather than per token file individually.
+
+## 5. Iterate over the platforms
 
 For each platform defined in your [config](config.md), Style Dictionary will do a few steps to get it ready to be consumed on that platform. You don't need to worry about one platform affecting another because everything that happens in a platform is non-destructive.
 
-## 4a. Transform the tokens
+## 5a. Transform the tokens
 
 Style Dictionary now traverses over the whole token object and looks for design tokens. It does this by looking for anything with a `value` key. When it comes across a design token, it then performs all the [transforms](transforms.md) defined in your [config](config.md) in order.
 
 Value transforms, transforms that modify a token's value, are skipped if the token references another token. Starting in 3.0, you can define a [transitive transform](transforms.md#transitive-transforms) that will transform a value that references another token after that reference has been resolved.
 
-## 4b. Resolve aliases / references to other values
+## 5b. Resolve aliases / references to other values
 
 After all the tokens have been transformed, it then does another pass over the token object looking for aliases, which look like `"{size.font.base.value}"`. When it finds these, it then replaces the reference with the transformed value. Because Style Dictionary merges all token files into a single object, aliases can be in any token file and still work.
 
-## 4c. Format the tokens into files
+## 5c. Format the tokens into files
 
 Now all the design tokens are ready to be written to a file. Style Dictionary takes the whole transformed and resolved token object and for each file defined in the platform it [formats](formats.md) the token object and write the output to a file. Internally, Style Dictionary creates a flat array of all the design tokens it finds in addition to the token object. This is how you can output a flat SCSS variables file.
 
-## 4d. Run actions
+## 5d. Run actions
 
 [Actions](actions.md) are custom code that run in a platform after the files are generated. They are useful for things like copying assets to specific build directories or generating images.
 

docs/config.md

@@ -85,7 +85,7 @@ module.exports = {
 
 Some interesting things you can do in a CommonJS file that you cannot do in a JSON file:
 
-- Add custom transforms, formats, filters, actions, and parsers
+- Add custom transforms, formats, filters, actions, preprocessors and parsers
 - Programmatically generate your configuration
 
 ---
@@ -145,17 +145,18 @@ You would then change your npm script or CLI command to run that file with Node:
 
 ## Attributes
 
-| Attribute | Type                     | Description                                                                                                                                                                                                                                                                      |
-| :-------- | :----------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| transform | Object (optional)        | Custom [transforms](transforms.md) you can include inline rather than using `.registerTransform`. The keys in this object will be the transform's name, the value should be an object with `type`                                                                                |
-| format    | Object (optional)        | Custom [formats](formats.md) you can include inline in the configuration rather than using `.registerFormat`. The keys in this object will be for format's name and value should be the formatter function.                                                                      |
-| action    | Object (optional)        | Custom inline [actions](actions.md). The keys in this object will be the action's name and the value should be an object containing `do` and `undo` methods.                                                                                                                     |
-| parsers   | Array[Parser] (optional) | Custom [file parsers](parsers.md) to run on input files                                                                                                                                                                                                                          |
-| include   | Array[String] (optional) | An array of file path [globs](https://github.com/isaacs/node-glob) to design token files that contain default styles. Style Dictionary uses this as a base collection of design tokens. The tokens found using the "source" attribute will overwrite tokens found using include. |
-| source    | Array[String]            | An array of file path [globs](https://github.com/isaacs/node-glob) to design token files. Style Dictionary will do a deep merge of all of the token files, allowing you to organize your files however you want.                                                                 |
-| tokens    | Object                   | The tokens object is a way to include inline design tokens as opposed to using the `source` and `include` arrays.                                                                                                                                                                |
-
-| platforms | Object[Platform] | An object containing [platform](#platform) config objects that describe how the Style Dictionary should build for that platform. You can add any arbitrary attributes on this object that will get passed to formats and actions (more on these in a bit). This is useful for things like build paths, name prefixes, variable names, etc. |
+| Attribute     | Type                     | Description                                                                                                                                                                                                                                                                                                                                |
+| :------------ | :----------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| transform     | Object (optional)        | Custom [transforms](transforms.md) you can include inline rather than using `.registerTransform`. The keys in this object will be the transform's name, the value should be an object with `type`                                                                                                                                          |
+| format        | Object (optional)        | Custom [formats](formats.md) you can include inline in the configuration rather than using `.registerFormat`. The keys in this object will be for format's name and value should be the formatter function.                                                                                                                                |
+| action        | Object (optional)        | Custom inline [actions](actions.md). The keys in this object will be the action's name and the value should be an object containing `do` and `undo` methods.                                                                                                                                                                               |
+| parsers       | Array[Parser] (optional) | Custom [file parsers](parsers.md) to run on input files                                                                                                                                                                                                                                                                                    |
+| preprocessors | Object (optional)        | Custom [preprocessors](preprocessors.md) to run on the full token dictionary, before any transforms run, can be registered using `.registerPreprocessor`. The keys in this object will be the preprocessor's name                                                                                                                          |
+| include       | Array[String] (optional) | An array of file path [globs](https://github.com/isaacs/node-glob) to design token files that contain default styles. Style Dictionary uses this as a base collection of design tokens. The tokens found using the "source" attribute will overwrite tokens found using include.                                                           |
+| source        | Array[String]            | An array of file path [globs](https://github.com/isaacs/node-glob) to design token files. Style Dictionary will do a deep merge of all of the token files, allowing you to organize your files however you want.                                                                                                                           |
+| tokens        | Object                   | The tokens object is a way to include inline design tokens as opposed to using the `source` and `include` arrays.                                                                                                                                                                                                                          |
+| properties    | Object                   | **DEPRECATED** The properties object has been renamed to `tokens`. Using the `properties` object will still work for backwards compatibility.                                                                                                                                                                                              |
+| platforms     | Object[Platform]         | An object containing [platform](#platform) config objects that describe how the Style Dictionary should build for that platform. You can add any arbitrary attributes on this object that will get passed to formats and actions (more on these in a bit). This is useful for things like build paths, name prefixes, variable names, etc. |
 
 ### Platform
 

docs/extending.md

@@ -11,6 +11,7 @@ There is a straightforward way to extend Style Dictionary to meet your needs - s
 - [registerTemplate](api.md#registertemplate) (deprecated)
 - [registerAction](api.md#registeraction)
 - [registerParser](api.md#registerparser)
+- [registerPreprocessor](api.md#registerpreprocessor)
 
 ## Extension Examples
 

docs/preprocessors.md

@@ -0,0 +1,96 @@
+# Preprocessors
+
+Starting in version 4.0, you can define custom preprocessors to process the dictionary object as a whole, after it all token files have been parsed and combined into one.
+This is useful if you want to do more complex transformations on the dictionary as a whole, when all other ways are not powerful enough.
+
+It should be clear that using this feature should be a last resort. Using custom parsers to parse per file or using transforms to do transformations on a per token basis,
+gives more granular control and reduces the risks of making mistakes. That said, preprocessing the full dictionary gives ultimate flexibility when needed.
+
+---
+
+## Preprocessor structure
+
+A preprocessor is an object with two props:
+
+- `name`: the name of the preprocessor
+- `preprocessor` a callback function that receives the dictionary as a parameter, and returns the processed dictionary
+
+```javascript
+const myPreprocessor = {
+  name: 'strip-third-party-meta',
+  preprocessor: (dictionary) => {
+    delete dictionary.thirdPartyMetadata;
+    return dictionary;
+  },
+};
+```
+
+Asynchronous callback functions are also supported, giving even more flexibility.
+
+```javascript
+const myPreprocessor = {
+  name: 'strip-props',
+  preprocessor: async (dictionary) => {
+    const propsToDelete = await someAPICall();
+
+    propsToDelete.forEach((propName) => {
+      delete dictionary[propName];
+    });
+
+    return dictionary;
+  },
+};
+```
+
+---
+
+## Using parsers
+
+First you will need to tell Style Dictionary about your parser. You can do this in two ways:
+
+1. Using the `.registerPreprocessor` method
+1. Inline in the [configuration](config.md)
+
+### .registerPreprocessor
+
+```javascript
+import StyleDictionary from 'style-dictionary';
+
+StyleDictionary.registerPreprocessor(myPreprocessor);
+```
+
+### Inline
+
+```javascript
+export default {
+  preprocessors: {
+    'strip-props': myPreprocessor,
+  },
+  // ... the rest of the configuration
+};
+```
+
+---
+
+## Preprocessor examples
+
+Stripping description property recursively in the entire dictionary object:
+
+```js
+StyleDictionary.registerPreprocessor({
+  name: 'strip-descriptions',
+  preprocessor: (dict) => {
+    // recursively traverse token objects and delete description props
+    function removeDescription(slice) {
+      delete slice.description;
+      Object.values(slice).forEach((value) => {
+        if (typeof value === 'object') {
+          removeDescription(value);
+        }
+      });
+      return slice;
+    }
+    return removeDescription(dict);
+  },
+});
+```