Merge pull request #2 from jordipuig37/release/0.1.3

Release 0.1.3

.github/workflows/build.yml

@@ -25,8 +25,8 @@ jobs:
       - name: Build distribution files
         run: python -m build
 
-      - name: Upload to testpypi
+      - name: Upload to PyPI
         env:
           TWINE_USERNAME: "__token__"
           TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
-        run: twine upload --repository testpypi dist/*
+        run: twine upload dist/*

.github/workflows/lint.yml

@@ -28,4 +28,4 @@ jobs:
         run: mypy .
 
       - name: Run flake8
-        run: flake8
+        run: flake8

README.md

@@ -1,11 +1,42 @@
 # Easy QL
 
-EasyQL is a SQLFluff plugin
+EasyQL is a SQLFluff plugin to lint custom rules.
 
 ## Installation
 
-Before the plugin is released as a PyPi package, to use it locally you can download the code in the repo and run the following command from the main directory:
+You can install the plugin with pip:
 
 ```sh
-pip install -e .
+pip install sqlfluff-plugin-easy-ql
+```
+
+Additionally you should add the following `.sqlfluff` file (also find it in the code provided in the repo).
+
+```
+[sqlfluff]
+exclude_rules = LT02
+
+[sqlfluff:layout:type:binary_operator]
+line_position = trailing
+```
+
+## Usage
+
+Once sqlfluff and this plugin are installed, call the linter normally and all the standard and custom rules will be checked.
+
+```sh
+sqlfluff lint --dialect snowflake .
+```
+
+Additionally, we added a command to analyze a tree of directories and store the linting results. The command `easy-ql lint` will lint all files with `.sql` extension and store the results for each file in another file named `<original_file_name>.lint.sql`.
+This command accepts three arguments:
+
+* --dialect: to indicate the dialect
+* --directory or --file to indicate the directory to be analized or a single file
+
+Examples:
+
+```sh
+easy-ql lint --dialect snowflake --directory sql_samples
+easy-ql lint --dialect tsql --file sql_samples/test1.sql
 ```

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sqlfluff-plugin-easy-ql"
-version = "0.1.2"
+version = "0.1.3"
 authors = [
   { name="Jordi Puig Rabat", email="[email protected]" },
 ]
@@ -33,3 +33,6 @@ where = ["src"]
 
 [project.entry-points."sqlfluff"]
 sqlfluff_easy_ql = "sqlfluff_easy_ql"
+
+[project.scripts]
+easy-ql = "sqlfluff_easy_ql.__main__:main"

requirements.txt

@@ -4,3 +4,5 @@ mypy
 pytest
 sqlfluff
 twine
+tqdm
+types-tqdm

src/sqlfluff_easy_ql/CV01.py

@@ -0,0 +1,29 @@
+"""EasyQL convention rule
+"""
+
+from sqlfluff.core.rules import (
+    BaseRule,
+    LintResult,
+    RuleContext
+)
+from sqlfluff.utils.functional import FunctionalContext
+from sqlfluff.core.rules.crawlers import SegmentSeekerCrawler
+
+
+class Rule_EasyQL_CV01(BaseRule):
+    """Prohibit the use of 'WHERE 1=1' in queries."""
+    groups = ("all",)
+    crawl_behaviour = SegmentSeekerCrawler({"where_clause"})
+    is_fix_compatible = False
+
+    def _eval(self, context: RuleContext):
+        text_segments = (
+            FunctionalContext(context)
+            .segment.children()
+        )
+
+        for idx, seg in enumerate(text_segments):
+            # Look for the pattern '1=1'
+            if "1=1" in seg.raw_upper.replace(" ", ""):
+                return LintResult(anchor=seg)
+        return None

src/sqlfluff_easy_ql/LT01.py

@@ -58,7 +58,8 @@ def _eval(self, context: RuleContext) -> List[LintResult]:
         obj_reference = context.segment.type.split("_")[1] + "_reference"
         table_name_idx, table_name_segment = next(
             ((idx, s) for idx, s in enumerate(segments)
-                if s.type == obj_reference or s.type == "function_name")
+                if s.type == obj_reference or s.type == "table_reference" or
+                s.type == "object_reference" or s.type == "function_name")
         )
 
         # assert that there is a newline and 4 spaces before the name

src/sqlfluff_easy_ql/LT03.py

@@ -0,0 +1,28 @@
+"""Implementation of custom rule for indentation in JOIN ... ON clauses."""
+
+from sqlfluff.core.rules import BaseRule, LintResult, RuleContext
+from sqlfluff.core.rules.crawlers import SegmentSeekerCrawler
+
+
+class Rule_EasyQL_LT03(BaseRule):
+    """JOIN and ON keyword should be in the same line.
+    """
+    groups = ("all",)
+    crawl_behaviour = SegmentSeekerCrawler({"join_clause"})
+
+    def __init__(self, *args, **kwargs):
+        """Overwrite __init__ to set config."""
+        super().__init__(*args, **kwargs)
+
+    def _eval(self, context: RuleContext):
+        """We should not JOIN .. ON differnet lines."""
+        on_appeared = False
+        for seg in context.segment.segments:
+            if seg.raw.lower() == "on":
+                on_appeared = True
+            if seg.raw == "\n" and not on_appeared:
+                desc = "Join and ON keyword should be at the same line."
+                return LintResult(
+                    anchor=seg,
+                    description=desc
+                )

src/sqlfluff_easy_ql/LT04.py

@@ -0,0 +1,44 @@
+"""Implementation of custom rule for indentation in JOIN ... ON clauses."""
+
+from sqlfluff.core.rules import BaseRule, LintResult, RuleContext
+from sqlfluff.core.rules.crawlers import SegmentSeekerCrawler
+
+
+class Rule_EasyQL_LT04(BaseRule):
+    """Conditions in JOIN ... ON clause should be in a new line and indented.
+    """
+    groups = ("all",)
+    crawl_behaviour = SegmentSeekerCrawler({"join_on_condition"})
+    is_fix_compatible = False
+
+    def __init__(self, *args, **kwargs):
+        """Overwrite __init__ to set config."""
+        super().__init__(*args, **kwargs)
+
+    def _eval(self, context: RuleContext):
+        """Conditions in JOIN ... ON clause should be in a new line and
+        indented.
+        """
+        segments = context.segment.segments
+        # import pdb; pdb.set_trace()
+        if segments[2].type != "newline":
+            # add to lint errors the segment
+            return [
+                LintResult(
+                    anchor=segments[0],
+                    description="Conditions in JOIN ... ON clause should be in a new line."  # noqa: E501
+                )
+            ]
+
+        elif segments[2].type == "newline" and \
+                (segments[3].type != "whitespace" or
+                 segments[3].type == "whitespace" and
+                 len(segments[3].raw) != 4):
+            return [
+                LintResult(
+                    anchor=segments[0],
+                    description="Conditions in JOIN ... ON clause should be in a new line and indented with 4 spaces."  # noqa: E501
+                )
+            ]
+
+        # TODO: check that all statements are properly indented

src/sqlfluff_easy_ql/__init__.py

@@ -19,15 +19,19 @@ def get_rules() -> List[Type[BaseRule]]:
     is available for the validation steps in the meta class.
     """
     # i.e. we DO recommend importing here:
-    from sqlfluff_easy_ql.rules import (
-        Rule_EasyQL_L002,
-        Rule_EasyQL_L003
-    )  # noqa: F811
-    from sqlfluff_easy_ql.LT02 import Rule_EasyQL_LT02
     from sqlfluff_easy_ql.LT01 import Rule_EasyQL_LT01
-
-    return [Rule_EasyQL_LT01, Rule_EasyQL_LT02,
-            Rule_EasyQL_L002, Rule_EasyQL_L003]
+    from sqlfluff_easy_ql.LT02 import Rule_EasyQL_LT02
+    from sqlfluff_easy_ql.LT03 import Rule_EasyQL_LT03
+    from sqlfluff_easy_ql.LT04 import Rule_EasyQL_LT04
+    from sqlfluff_easy_ql.CV01 import Rule_EasyQL_CV01
+
+    return [
+        Rule_EasyQL_LT01,
+        Rule_EasyQL_LT02,
+        Rule_EasyQL_LT03,
+        Rule_EasyQL_LT04,
+        Rule_EasyQL_CV01
+    ]
 
 
 @hookimpl

src/sqlfluff_easy_ql/__main__.py

@@ -0,0 +1,112 @@
+import argparse
+import sys
+import os
+import subprocess
+
+from tqdm import tqdm
+from pathlib import Path
+
+
+def write_results(
+        result: subprocess.CompletedProcess[str],
+        lint_output_path: str
+) -> None:
+    """
+    Writes sqfluff output to a file. If the file already exists
+    the file is rewritten. If the process ends without lint errors,
+    the file is not written and if it exists the file is deleted.
+    """
+    if result.returncode != 0:
+        with open(lint_output_path, "w") as output_file:
+            output_file.write(result.stdout)
+    else:
+        if os.path.exists(lint_output_path):
+            os.remove(lint_output_path)
+
+
+def lint_sql_files(directory: str, dialect: str):
+    tree = list(os.walk(directory))
+    for root, _, files in tqdm(
+        tree, desc="Processing directories", position=0
+    ):
+        for file in tqdm(
+            files, desc=f"Processing files in {root}", position=1, leave=False
+        ):
+            if Path(file).suffix.lower() == ".sql":
+                file_path = os.path.join(root, file)
+                file_name = Path(file_path).stem
+                lint_output_path = f"{file_name}.lint.txt"
+
+                # Run sqlfluff lint and capturing the output
+                result = subprocess.run([
+                    "python", "-m", "sqlfluff", "lint", "--dialect", dialect,
+                    file_path],
+                    capture_output=True, text=True
+                )
+                write_results(result, lint_output_path)
+
+
+def lint_single_sql_file(file_path: str, dialect: str):
+    if Path(file_path).suffix.lower() == ".sql":
+        file_name = file_name = Path(file_path).stem
+        lint_output_path = f"{file_name}.lint.txt"
+
+        result = subprocess.run([
+            "python", "-m", "sqlfluff", "lint", "--dialect", dialect,
+            file_path],
+            capture_output=True, text=True
+        )
+        write_results(result, lint_output_path)
+
+
+def lint(args):
+    """Main linter adapted to easy-ql.
+
+    This function will lint the files (either a single one or all files in a
+    directory) and store the lint results in a new file with the name pattern:
+    <original_file_name>.lint.txt
+
+    Arguments include: (see main)
+        + dialect
+        + file
+        + directory
+    """
+    if args.file != "":
+        lint_single_sql_file(args.file, args.dialect)
+    else:
+        lint_sql_files(args.directory, args.dialect)
+
+
+def main():
+    parser = argparse.ArgumentParser(prog='easy-ql')
+    subparsers = parser.add_subparsers(dest='command')
+
+    parser_lint = subparsers.add_parser(
+        'lint', help='Run the lint and save the results in files'
+    )
+    parser_lint.add_argument(
+        '--directory',
+        type=str, default=".",
+        help="If this option is set, lint all .sql files in the directory."
+    )
+    parser_lint.add_argument(
+        "-f", '--file',
+        type=str, default="",
+        help="If this option is set, lint the file."
+    )
+    parser_lint.add_argument(
+        '--dialect',
+        type=str, default="ansi",
+        help="The dialect used to lint the files."
+    )
+    parser_lint.set_defaults(func=lint)
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
+    # Call the appropriate function
+    args.func(args)