Add flake8 pre-commit hook

.pre-commit-config.yaml

@@ -4,6 +4,11 @@ repos:
     hooks:
       - id: black
         language_version: python3
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        language_version: python3
   - repo: https://github.com/pycqa/isort
     rev: 5.7.0
     hooks:

dask_sql/context.py

@@ -2,8 +2,7 @@
 import inspect
 import logging
 import warnings
-from collections import namedtuple
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, Union
 
 import dask.dataframe as dd
 import pandas as pd
@@ -30,6 +29,9 @@
 from dask_sql.physical.rex import RexConverter, core
 from dask_sql.utils import ParsingException
 
+if TYPE_CHECKING:
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 
@@ -600,7 +602,7 @@ def stop_server(self):  # pragma: no cover
         """
         Stop a SQL server started by ``run_server`.
         """
-        if not self.sql_server is None:
+        if self.sql_server is not None:
             loop = asyncio.get_event_loop()
             assert loop
             loop.create_task(self.sql_server.shutdown())
@@ -767,7 +769,8 @@ def _to_sql_string(self, s: "org.apache.calcite.sql.SqlNode", default_dialect=No
 
         try:
             return str(s.toSqlString(default_dialect))
-        except:  # pragma: no cover. Have not seen any instance so far, but better be safe than sorry.
+        # Have not seen any instance so far, but better be safe than sorry
+        except Exception:  # pragma: no cover
             return str(s)
 
     def _get_tables_from_stack(self):

dask_sql/input_utils/convert.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Union
+from typing import TYPE_CHECKING, Union
 
 import dask.dataframe as dd
 import pandas as pd
@@ -8,6 +8,11 @@
 from dask_sql.input_utils.base import BaseInputPlugin
 from dask_sql.utils import Pluggable
 
+if TYPE_CHECKING:
+    import cudf
+    import hive
+    import sqlalchemy
+
 logger = logging.Logger(__name__)
 
 InputType = Union[

dask_sql/input_utils/hive.py

@@ -126,7 +126,7 @@ def wrapped_read_function(location, column_information, **kwargs):
             else:  # pragma: no cover
                 # prevent python to optimize it away and make coverage not respect the
                 # pragma
-                dummy = 0
+                dummy = 0  # noqa: F841
             df = read_function(location, **kwargs)
 
             logger.debug(f"Applying column information: {column_information}")
@@ -251,7 +251,7 @@ def _parse_hive_table_description(
                 else:  # pragma: no cover
                     # prevent python to optimize it away and make coverage not respect the
                     # pragma
-                    dummy = 0
+                    dummy = 0  # noqa: F841
             elif value and last_field is not None:
                 last_field[value] = value2
 

dask_sql/integrations/ipython.py

@@ -1,9 +1,12 @@
 import json
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List
 
 from dask_sql.mappings import _SQL_TO_PYTHON_FRAMES
 from dask_sql.physical.rex.core import RexCallPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+
 # JS snippet to use the created mime type highlighthing
 _JS_ENABLE_DASK_SQL = r"""
 require(['notebook/js/codecell'], function(codecell) {
@@ -128,6 +131,6 @@ def _register_syntax_highlighting():  # pragma: no cover
     display.display_javascript(js + _JS_ENABLE_DASK_SQL, raw=True)
 
 
-def _create_set(l: List[str]) -> Dict[str, bool]:  # pragma: no cover
+def _create_set(keys: List[str]) -> Dict[str, bool]:  # pragma: no cover
     """Small helper function to turn a list into the correct format for codemirror"""
-    return {key: True for key in l}
+    return {key: True for key in keys}

dask_sql/java.py

@@ -55,7 +55,7 @@ def _set_or_check_java_home():
     from pyarrow.hdfs import _maybe_set_hadoop_classpath
 
     _maybe_set_hadoop_classpath()
-except:  # pragma: no cover
+except Exception:  # pragma: no cover
     pass
 
 # Define how to run the java virtual machine.

dask_sql/physical/rel/base.py

@@ -1,11 +1,15 @@
 import logging
-from typing import List
+from typing import TYPE_CHECKING, List
 
 import dask.dataframe as dd
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.mappings import cast_column_type, sql_to_python_type
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/convert.py

@@ -1,11 +1,16 @@
 import logging
+from typing import TYPE_CHECKING
 
 import dask.dataframe as dd
 
 from dask_sql.java import get_java_class
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import LoggableDataFrame, Pluggable
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/analyze.py

@@ -1,10 +1,16 @@
+from typing import TYPE_CHECKING
+
 import dask.dataframe as dd
 import pandas as pd
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.mappings import python_to_sql_type
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 
 class AnalyzeTablePlugin(BaseRelPlugin):
     """

dask_sql/physical/rel/custom/columns.py

@@ -1,10 +1,16 @@
+from typing import TYPE_CHECKING
+
 import dask.dataframe as dd
 import pandas as pd
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.mappings import python_to_sql_type
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 
 class ShowColumnsPlugin(BaseRelPlugin):
     """

dask_sql/physical/rel/custom/create_experiment.py

@@ -1,4 +1,5 @@
 import logging
+from typing import TYPE_CHECKING
 
 import dask.dataframe as dd
 import pandas as pd
@@ -8,6 +9,9 @@
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import convert_sql_kwargs, import_class
 
+if TYPE_CHECKING:
+    import dask_sql
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/create_model.py

@@ -1,10 +1,14 @@
 import logging
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.java import org
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import convert_sql_kwargs, import_class
 
+if TYPE_CHECKING:
+    import dask_sql
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/create_schema.py

@@ -1,8 +1,12 @@
 import logging
+from typing import TYPE_CHECKING
 
-from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/create_table.py

@@ -1,9 +1,14 @@
 import logging
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import convert_sql_kwargs
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/create_table_as.py

@@ -1,8 +1,13 @@
 import logging
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/describe_model.py

@@ -1,9 +1,15 @@
+from typing import TYPE_CHECKING
+
 import dask.dataframe as dd
 import pandas as pd
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 
 class ShowModelParamsPlugin(BaseRelPlugin):
     """

dask_sql/physical/rel/custom/drop_model.py

@@ -1,8 +1,13 @@
 import logging
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/drop_schema.py

@@ -1,8 +1,12 @@
 import logging
+from typing import TYPE_CHECKING
 
-from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/drop_table.py

@@ -1,8 +1,13 @@
 import logging
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/export_model.py

@@ -1,9 +1,14 @@
 import logging
 import pickle
+from typing import TYPE_CHECKING
 
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import convert_sql_kwargs
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 logger = logging.getLogger(__name__)
 
 
@@ -67,7 +72,7 @@ def convert(
                 import mlflow
             except ImportError:  # pragma: no cover
                 raise ImportError(
-                    f"For export in the mlflow format, you need to have mlflow installed"
+                    "For export in the mlflow format, you need to have mlflow installed"
                 )
             try:
                 import sklearn
@@ -77,7 +82,7 @@ def convert(
                 mlflow.sklearn.save_model(model, location, **kwargs)
             else:
                 raise NotImplementedError(
-                    f"dask-sql supports only sklearn compatible model i.e fit-predict style model"
+                    "dask-sql supports only sklearn compatible model i.e fit-predict style model"
                 )
         elif format == "onnx":
             """

dask_sql/physical/rel/custom/predict.py

@@ -1,11 +1,15 @@
 import copy
 import logging
 import uuid
+from typing import TYPE_CHECKING
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.java import com, java, org
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+
 logger = logging.getLogger(__name__)
 
 

dask_sql/physical/rel/custom/schemas.py

@@ -1,9 +1,15 @@
+from typing import TYPE_CHECKING
+
 import dask.dataframe as dd
 import pandas as pd
 
 from dask_sql.datacontainer import ColumnContainer, DataContainer
 from dask_sql.physical.rel.base import BaseRelPlugin
 
+if TYPE_CHECKING:
+    import dask_sql
+    from dask_sql.java import org
+
 
 class ShowSchemasPlugin(BaseRelPlugin):
     """