Add flake8 pre-commit hook

.pre-commit-config.yaml

@@ -4,6 +4,11 @@ repos:
     hooks:
       - id: black
         language_version: python3
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        language_version: python3
   - repo: https://github.com/pycqa/isort
     rev: 5.7.0
     hooks:

conftest.py

@@ -1,3 +1 @@
-import pytest
-
 pytest_plugins = ["distributed.utils_test", "tests.integration.fixtures"]

dask_sql/context.py

@@ -2,7 +2,6 @@
 import inspect
 import logging
 import warnings
-from collections import namedtuple
 from typing import Any, Callable, Dict, List, Tuple, Union
 
 import dask.dataframe as dd
@@ -600,7 +599,7 @@ def stop_server(self):  # pragma: no cover
         """
         Stop a SQL server started by ``run_server`.
         """
-        if not self.sql_server is None:
+        if self.sql_server is not None:
             loop = asyncio.get_event_loop()
             assert loop
             loop.create_task(self.sql_server.shutdown())
@@ -767,7 +766,8 @@ def _to_sql_string(self, s: "org.apache.calcite.sql.SqlNode", default_dialect=No
 
         try:
             return str(s.toSqlString(default_dialect))
-        except:  # pragma: no cover. Have not seen any instance so far, but better be safe than sorry.
+        # Have not seen any instance so far, but better be safe than sorry
+        except:  # noqa: E722; pragma: no cover
             return str(s)
 
     def _get_tables_from_stack(self):

dask_sql/input_utils/hive.py

@@ -126,7 +126,7 @@ def wrapped_read_function(location, column_information, **kwargs):
             else:  # pragma: no cover
                 # prevent python to optimize it away and make coverage not respect the
                 # pragma
-                dummy = 0
+                dummy = 0  # noqa: F841
             df = read_function(location, **kwargs)
 
             logger.debug(f"Applying column information: {column_information}")
@@ -251,7 +251,7 @@ def _parse_hive_table_description(
                 else:  # pragma: no cover
                     # prevent python to optimize it away and make coverage not respect the
                     # pragma
-                    dummy = 0
+                    dummy = 0  # noqa: F841
             elif value and last_field is not None:
                 last_field[value] = value2
 

dask_sql/integrations/ipython.py

@@ -128,6 +128,6 @@ def _register_syntax_highlighting():  # pragma: no cover
     display.display_javascript(js + _JS_ENABLE_DASK_SQL, raw=True)
 
 
-def _create_set(l: List[str]) -> Dict[str, bool]:  # pragma: no cover
+def _create_set(keys: List[str]) -> Dict[str, bool]:  # pragma: no cover
     """Small helper function to turn a list into the correct format for codemirror"""
-    return {key: True for key in l}
+    return {key: True for key in keys}

dask_sql/physical/rel/custom/create_schema.py

@@ -1,6 +1,5 @@
 import logging
 
-from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
 logger = logging.getLogger(__name__)

dask_sql/physical/rel/custom/drop_schema.py

@@ -1,6 +1,5 @@
 import logging
 
-from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
 logger = logging.getLogger(__name__)

dask_sql/physical/rel/custom/export_model.py

@@ -67,7 +67,7 @@ def convert(
                 import mlflow
             except ImportError:  # pragma: no cover
                 raise ImportError(
-                    f"For export in the mlflow format, you need to have mlflow installed"
+                    "For export in the mlflow format, you need to have mlflow installed"
                 )
             try:
                 import sklearn
@@ -77,7 +77,7 @@ def convert(
                 mlflow.sklearn.save_model(model, location, **kwargs)
             else:
                 raise NotImplementedError(
-                    f"dask-sql supports only sklearn compatible model i.e fit-predict style model"
+                    "dask-sql supports only sklearn compatible model i.e fit-predict style model"
                 )
         elif format == "onnx":
             """

dask_sql/physical/rel/custom/switch_schema.py

@@ -1,4 +1,4 @@
-from dask_sql.datacontainer import ColumnContainer, DataContainer
+from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
 

dask_sql/physical/rel/logical/aggregate.py

@@ -2,7 +2,7 @@
 import operator
 from collections import defaultdict
 from functools import reduce
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Tuple
 
 import dask.dataframe as dd
 import pandas as pd

dask_sql/physical/rel/logical/project.py

@@ -1,11 +1,9 @@
 import logging
-from uuid import uuid4
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.java import org
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.physical.rex import RexConverter
-from dask_sql.physical.rex.core.input_ref import RexInputRefPlugin
 from dask_sql.utils import new_temporary_column
 
 logger = logging.getLogger(__name__)

dask_sql/physical/rel/logical/sort.py

@@ -1,6 +1,3 @@
-from typing import List
-
-import dask
 import dask.dataframe as dd
 
 from dask_sql.datacontainer import DataContainer
@@ -9,7 +6,6 @@
 from dask_sql.physical.rex import RexConverter
 from dask_sql.physical.utils.map import map_on_partition_index
 from dask_sql.physical.utils.sort import apply_sort
-from dask_sql.utils import new_temporary_column
 
 
 class LogicalSortPlugin(BaseRelPlugin):

dask_sql/physical/rel/logical/table_scan.py

@@ -1,5 +1,3 @@
-from typing import Dict
-
 from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 

dask_sql/physical/rel/logical/union.py

@@ -2,7 +2,6 @@
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
-from dask_sql.physical.rex import RexConverter
 
 
 class LogicalUnionPlugin(BaseRelPlugin):

dask_sql/physical/rel/logical/window.py

@@ -1,7 +1,7 @@
 import logging
 from collections import namedtuple
 from functools import partial
-from typing import Any, Callable, List, Optional, Tuple
+from typing import Callable, List, Optional, Tuple
 
 import dask.dataframe as dd
 import numpy as np
@@ -14,7 +14,6 @@
 from dask_sql.physical.rex.convert import RexConverter
 from dask_sql.physical.rex.core.literal import RexLiteralPlugin
 from dask_sql.physical.utils.groupby import get_groupby_with_nulls_cols
-from dask_sql.physical.utils.map import map_on_partition_index
 from dask_sql.physical.utils.sort import sort_partition_func
 from dask_sql.utils import (
     LoggableDataFrame,
@@ -96,7 +95,7 @@ def to_bound_description(
         else:  # pragma: no cover
             # prevent python to optimize it away and make coverage not respect the
             # pragma
-            dummy = 0
+            dummy = 0  # noqa: F841
         offset = int(RexLiteralPlugin().convert(offset, None, None))
     else:
         offset = None

dask_sql/physical/rex/core/call.py

@@ -476,7 +476,6 @@ def __init__(self):
         super().__init__(self.extract)
 
     def extract(self, what, df: SeriesOrScalar):
-        input_df = df
         df = convert_to_datetime(df)
 
         if what == "CENTURY":
@@ -522,7 +521,7 @@ def __init__(self, round_method: str):
         assert round_method in {
             "ceil",
             "floor",
-        }, f"Round method can only be either ceil or floor"
+        }, "Round method can only be either ceil or floor"
 
         super().__init__(
             is_datetime,  # if the series is dt type

dask_sql/physical/rex/core/literal.py

@@ -1,7 +1,6 @@
 from typing import Any
 
 import dask.dataframe as dd
-import numpy as np
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.java import com, org

dask_sql/physical/utils/sort.py

@@ -3,7 +3,7 @@
 import dask.dataframe as dd
 import pandas as pd
 
-from dask_sql.utils import make_pickable_without_dask_sql, new_temporary_column
+from dask_sql.utils import make_pickable_without_dask_sql
 
 
 def apply_sort(

dask_sql/server/responses.py

@@ -1,9 +1,8 @@
 import uuid
-from typing import List
 
 import dask.dataframe as dd
 import numpy as np
-from fastapi import FastAPI, Request
+from fastapi import Request
 
 from dask_sql.mappings import python_to_sql_type
 

dask_sql/utils.py

@@ -1,11 +1,9 @@
 import importlib
 import logging
 import re
-import sys
 from collections import defaultdict
-from contextlib import contextmanager
 from datetime import datetime
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict
 from unittest.mock import patch
 from uuid import uuid4
 

setup.cfg

@@ -0,0 +1,21 @@
+[flake8]
+# References:
+# https://flake8.readthedocs.io/en/latest/user/configuration.html
+# https://flake8.readthedocs.io/en/latest/user/error-codes.html
+# https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
+exclude = __init__.py,versioneer.py
+ignore =
+    E203,  # whitespace before ':'
+    E231,E241,  # Multiple spaces around ","
+    E731,  # Assigning lambda expression
+    #E741,  # Ambiguous variable names
+    W503,  # line break before binary operator
+    W504,  # line break after binary operator
+    F821,  # undefined name
+per-file-ignores =
+    tests/*:
+        # local variable is assigned to but never used
+        F841,
+        # Ambiguous variable name
+        E741,
+max-line-length = 150

tests/integration/test_create.py

@@ -35,7 +35,7 @@ def test_cluster_memory(client, c, df):
     client.publish_dataset(df=dd.from_pandas(df, npartitions=1))
 
     c.sql(
-        f"""
+        """
         CREATE TABLE
             new_table
         WITH (
@@ -82,7 +82,7 @@ def test_create_from_csv_persist(c, df, temporary_data_file):
 def test_wrong_create(c):
     with pytest.raises(AttributeError):
         c.sql(
-            f"""
+            """
             CREATE TABLE
                 new_table
             WITH (
@@ -93,7 +93,7 @@ def test_wrong_create(c):
 
     with pytest.raises(AttributeError):
         c.sql(
-            f"""
+            """
             CREATE TABLE
                 new_table
             WITH (
@@ -106,7 +106,7 @@ def test_wrong_create(c):
 
 def test_create_from_query(c, df):
     c.sql(
-        f"""
+        """
         CREATE OR REPLACE TABLE
             new_table
         AS (
@@ -124,7 +124,7 @@ def test_create_from_query(c, df):
     assert_frame_equal(df, return_df)
 
     c.sql(
-        f"""
+        """
         CREATE OR REPLACE VIEW
             new_table
         AS (
@@ -159,7 +159,7 @@ def test_view_table_persist(c, temporary_data_file, df):
     # Views should change, when the original data changes
     # Tables should not change, when the original data changes
     c.sql(
-        f"""
+        """
         CREATE VIEW
             count_view
         AS (
@@ -168,7 +168,7 @@ def test_view_table_persist(c, temporary_data_file, df):
     """
     )
     c.sql(
-        f"""
+        """
         CREATE TABLE
             count_table
         AS (
@@ -196,7 +196,7 @@ def test_view_table_persist(c, temporary_data_file, df):
 
 def test_replace_and_error(c, temporary_data_file, df):
     c.sql(
-        f"""
+        """
         CREATE TABLE
             new_table
         AS (
@@ -213,7 +213,7 @@ def test_replace_and_error(c, temporary_data_file, df):
 
     with pytest.raises(RuntimeError):
         c.sql(
-            f"""
+            """
             CREATE TABLE
                 new_table
             AS (
@@ -223,7 +223,7 @@ def test_replace_and_error(c, temporary_data_file, df):
         )
 
     c.sql(
-        f"""
+        """
         CREATE TABLE IF NOT EXISTS
             new_table
         AS (
@@ -239,7 +239,7 @@ def test_replace_and_error(c, temporary_data_file, df):
     )
 
     c.sql(
-        f"""
+        """
         CREATE OR REPLACE TABLE
             new_table
         AS (
@@ -260,7 +260,7 @@ def test_replace_and_error(c, temporary_data_file, df):
         c.sql("SELECT a FROM new_table")
 
     c.sql(
-        f"""
+        """
         CREATE TABLE IF NOT EXISTS
             new_table
         AS (
@@ -332,7 +332,7 @@ def test_drop(c):
     c.sql("DROP TABLE IF EXISTS new_table")
 
     c.sql(
-        f"""
+        """
         CREATE TABLE
             new_table
         AS (