Fix pylint

.github/workflows/test.yml

@@ -33,7 +33,37 @@ jobs:
     - name: Install Qlib with pip
       run: |
         pip install numpy==1.19.5 ruamel.yaml
-        pip install pyqlib --ignore-installed 
+        pip install pyqlib --ignore-installed
+
+    # Check Qlib with pylint
+    # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
+      # C0103: invalid-name
+      # C0209: consider-using-f-string
+      # R0402: consider-using-from-import
+      # R1705: no-else-return
+      # R1710: inconsistent-return-statements
+      # R1725: super-with-arguments
+      # R1735: use-dict-literal
+      # W0102: dangerous-default-value
+      # W0212: protected-access
+      # W0221: arguments-differ
+      # W0223: abstract-method
+      # W0231: super-init-not-called
+      # W0237: arguments-renamed
+      # W0612: unused-variable
+      # W0621: redefined-outer-name
+      # W0622: redefined-builtin
+      # FIXME: specify exception type
+      # W0703: broad-except
+      # W1309: f-string-without-interpolation
+      # E1102: not-callable
+      # E1136: unsubscriptable-object
+    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962 
+    - name: Check Qlib with pylint
+      run: |
+        pip install --upgrade pip
+        pip install pylint
+        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0201,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
 
     - name: Test data downloads
       run: |

qlib/__init__.py

@@ -30,8 +30,8 @@ def init(default_conf="client", **kwargs):
             When using the recorder, skip_if_reg can set to True to avoid loss of recorder.
 
     """
-    from .config import C
-    from .data.cache import H
+    from .config import C  # pylint: disable=C0415
+    from .data.cache import H  # pylint: disable=C0415
 
     # FIXME: this logger ignored the level in config
     logger = get_module_logger("Initialization", level=logging.INFO)
@@ -85,7 +85,7 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
     mount_command = "sudo mount.nfs %s %s" % (provider_uri, mount_path)
     # If the provider uri looks like this 172.23.233.89//data/csdesign'
     # It will be a nfs path. The client provider will be used
-    if not auto_mount:
+    if not auto_mount:  # pylint: disable=R1702
         if not Path(mount_path).exists():
             raise FileNotFoundError(
                 f"Invalid mount path: {mount_path}! Please mount manually: {mount_command} or Set init parameter `auto_mount=True`"
@@ -139,8 +139,10 @@ def _mount_nfs_uri(provider_uri, mount_path, auto_mount: bool = False):
             if not _is_mount:
                 try:
                     Path(mount_path).mkdir(parents=True, exist_ok=True)
-                except Exception:
-                    raise OSError(f"Failed to create directory {mount_path}, please create {mount_path} manually!")
+                except Exception as e:
+                    raise OSError(
+                        f"Failed to create directory {mount_path}, please create {mount_path} manually!"
+                    ) from e
 
                 # check nfs-common
                 command_res = os.popen("dpkg -l | grep nfs-common")

qlib/backtest/__init__.py

@@ -171,8 +171,8 @@ def get_strategy_executor(
     # NOTE:
     # - for avoiding recursive import
     # - typing annotations is not reliable
-    from ..strategy.base import BaseStrategy
-    from .executor import BaseExecutor
+    from ..strategy.base import BaseStrategy  # pylint: disable=C0415
+    from .executor import BaseExecutor  # pylint: disable=C0415
 
     trade_account = create_account_instance(
         start_time=start_time, end_time=end_time, benchmark=benchmark, account=account, pos_type=pos_type

qlib/backtest/account.py

@@ -2,11 +2,11 @@
 # Licensed under the MIT License.
 from __future__ import annotations
 import copy
-from typing import Dict, List, Tuple, TYPE_CHECKING
+from typing import Dict, List, Tuple
 from qlib.utils import init_instance_by_config
 import pandas as pd
 
-from .position import BasePosition, InfPosition, Position
+from .position import BasePosition
 from .report import PortfolioMetrics, Indicator
 from .decision import BaseTradeDecision, Order
 from .exchange import Exchange

qlib/backtest/decision.py

@@ -7,19 +7,18 @@
 from qlib.utils.time import concat_date_time, epsilon_change
 from qlib.log import get_module_logger
 
+from typing import ClassVar, Optional, Union, List, Tuple
+
 # try to fix circular imports when enabling type hints
-from typing import Callable, TYPE_CHECKING
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from qlib.strategy.base import BaseStrategy
     from qlib.backtest.exchange import Exchange
 from qlib.backtest.utils import TradeCalendarManager
-import warnings
 import numpy as np
 import pandas as pd
-import numpy as np
-from dataclasses import dataclass, field
-from typing import ClassVar, Optional, Union, List, Set, Tuple
+from dataclasses import dataclass
 
 
 class OrderDir(IntEnum):
@@ -418,7 +417,7 @@ def get_range_limit(self, **kwargs) -> Tuple[int, int]:
                 return kwargs["default_value"]
             else:
                 # Default to get full index
-                raise NotImplementedError(f"The decision didn't provide an index range")
+                raise NotImplementedError(f"The decision didn't provide an index range") from NotImplementedError
 
         # clip index
         if getattr(self, "total_step", None) is not None:

qlib/backtest/exchange.py

@@ -3,13 +3,13 @@
 from __future__ import annotations
 from collections import defaultdict
 from typing import TYPE_CHECKING
+from typing import List, Tuple, Union
 
 if TYPE_CHECKING:
     from .account import Account
 
 from qlib.backtest.position import BasePosition, Position
 import random
-from typing import List, Tuple, Union
 import numpy as np
 import pandas as pd
 
@@ -18,7 +18,7 @@
 from ..constant import REG_CN
 from ..log import get_module_logger
 from .decision import Order, OrderDir, OrderHelper
-from .high_performance_ds import BaseQuote, PandasQuote, NumpyQuote
+from .high_performance_ds import BaseQuote, NumpyQuote
 
 
 class Exchange:

qlib/backtest/executor.py

@@ -1,22 +1,18 @@
-from abc import abstractclassmethod, abstractmethod
+from abc import abstractmethod
 import copy
 from qlib.backtest.position import BasePosition
 from qlib.log import get_module_logger
 from types import GeneratorType
 from qlib.backtest.account import Account
-import warnings
 import pandas as pd
 from typing import List, Tuple, Union
 from collections import defaultdict
 
-from qlib.backtest.report import Indicator
-
-from .decision import EmptyTradeDecision, Order, BaseTradeDecision
+from .decision import Order, BaseTradeDecision
 from .exchange import Exchange
 from .utils import TradeCalendarManager, CommonInfrastructure, LevelInfrastructure, get_start_end_idx
 
 from ..utils import init_instance_by_config
-from ..utils.time import Freq
 from ..strategy.base import BaseStrategy
 
 
@@ -193,7 +189,8 @@ def execute(self, trade_decision: BaseTradeDecision, level: int = 0):
             pass
         return return_value.get("execute_result")
 
-    @abstractclassmethod
+    @classmethod
+    @abstractmethod
     def _collect_data(cls, trade_decision: BaseTradeDecision, level: int = 0) -> Tuple[List[object], dict]:
         """
         Please refer to the doc of collect_data
@@ -453,7 +450,6 @@ def post_inner_exe_step(self, inner_exe_res):
         inner_exe_res :
             the execution result of inner task
         """
-        pass
 
     def get_all_executors(self):
         """get all executors, including self and inner_executor.get_all_executors()"""

qlib/backtest/position.py

@@ -2,8 +2,6 @@
 # Licensed under the MIT License.
 
 
-import copy
-import pathlib
 from typing import Dict, List, Union
 
 import pandas as pd
@@ -538,7 +536,7 @@ def get_cash(self, include_settle=False) -> float:
     def get_stock_amount_dict(self) -> Dict:
         raise NotImplementedError(f"InfPosition doesn't support get_stock_amount_dict")
 
-    def get_stock_weight_dict(self, only_stock: bool) -> Dict:
+    def get_stock_weight_dict(self, only_stock: bool = False) -> Dict:
         raise NotImplementedError(f"InfPosition doesn't support get_stock_weight_dict")
 
     def add_count_all(self, bar):

qlib/backtest/report.py

@@ -10,11 +10,8 @@
 import pandas as pd
 
 from qlib.backtest.exchange import Exchange
-from .decision import IdxTradeRange
 from qlib.backtest.decision import BaseTradeDecision, Order, OrderDir
-from qlib.backtest.utils import TradeCalendarManager
-from .high_performance_ds import BaseOrderIndicator, PandasOrderIndicator, NumpyOrderIndicator, SingleMetric
-from ..data import D
+from .high_performance_ds import BaseOrderIndicator, NumpyOrderIndicator, SingleMetric
 from ..tests.config import CSI300_BENCH
 from ..utils.resam import get_higher_eq_freq_feature, resam_ts_data
 import qlib.utils.index_data as idd

qlib/config.py

@@ -388,13 +388,11 @@ def set(self, default_conf: str = "client", **kwargs):
         default_conf : str
             the default config template chosen by user: "server", "client"
         """
-        from .utils import set_log_with_config, get_module_logger, can_use_cache
+        from .utils import set_log_with_config, get_module_logger, can_use_cache  # pylint: disable=C0415
 
         self.reset()
 
-        _logging_config = self.logging_config
-        if "logging_config" in kwargs:
-            _logging_config = kwargs["logging_config"]
+        _logging_config = kwargs.get("logging_config", self.logging_config)
 
         # set global config
         if _logging_config:
@@ -433,11 +431,11 @@ def set(self, default_conf: str = "client", **kwargs):
                     )
 
     def register(self):
-        from .utils import init_instance_by_config
-        from .data.ops import register_all_ops
-        from .data.data import register_all_wrappers
-        from .workflow import R, QlibRecorder
-        from .workflow.utils import experiment_exit_handler
+        from .utils import init_instance_by_config  # pylint: disable=C0415
+        from .data.ops import register_all_ops  # pylint: disable=C0415
+        from .data.data import register_all_wrappers  # pylint: disable=C0415
+        from .workflow import R, QlibRecorder  # pylint: disable=C0415
+        from .workflow.utils import experiment_exit_handler  # pylint: disable=C0415
 
         register_all_ops(self)
         register_all_wrappers(self)
@@ -454,7 +452,7 @@ def register(self):
         self._registered = True
 
     def reset_qlib_version(self):
-        import qlib
+        import qlib  # pylint: disable=C0415
 
         reset_version = self.get("qlib_reset_version", None)
         if reset_version is not None:

qlib/contrib/data/dataset.py

@@ -7,16 +7,15 @@
 import numpy as np
 import pandas as pd
 
-from qlib.utils import init_instance_by_config
-from qlib.data.dataset import DatasetH, DataHandler
+from qlib.data.dataset import DatasetH
 
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
 def _to_tensor(x):
     if not isinstance(x, torch.Tensor):
-        return torch.tensor(x, dtype=torch.float, device=device)
+        return torch.tensor(x, dtype=torch.float, device=device)  # pylint: disable=E1101
     return x
 
 

qlib/contrib/data/handler.py

@@ -5,9 +5,7 @@
 from ...data.dataset.processor import Processor
 from ...utils import get_callable_kwargs
 from ...data.dataset import processor as processor_module
-from ...log import TimeInspector
 from inspect import getfullargspec
-import copy
 
 
 def check_transform_proc(proc_l, fit_start_time, fit_end_time):

qlib/contrib/data/processor.py

@@ -1,9 +1,6 @@
 import numpy as np
-import pandas as pd
-import copy
 
 from ...log import TimeInspector
-from ...utils.serial import Serializable
 from ...data.dataset.processor import Processor, get_group_columns
 
 

qlib/contrib/evaluate_portfolio.py

@@ -5,12 +5,10 @@
 from __future__ import division
 from __future__ import print_function
 
-import copy
 import numpy as np
 import pandas as pd
 from scipy.stats import spearmanr, pearsonr
 
-
 from ..data import D
 
 from collections import OrderedDict
@@ -243,4 +241,4 @@ def get_rank_ic(a, b):
 
 
 def get_normal_ic(a, b):
-    return pearsonr(a, b).correlation
+    return pearsonr(a, b)[0]

qlib/contrib/meta/data_selection/dataset.py

@@ -1,24 +1,23 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
+import pandas as pd
+import numpy as np
 from copy import deepcopy
+from joblib import Parallel, delayed  # pylint: disable=E0401
+from typing import Dict, List, Union, Text, Tuple
 from qlib.data.dataset.utils import init_task_handler
-from qlib.utils.data import deepcopy_basic_type
+from qlib.data.dataset import DatasetH
 from qlib.contrib.torch import data_to_tensor
-from qlib.workflow.task.utils import TimeAdjuster
 from qlib.model.meta.task import MetaTask
-from typing import Dict, List, Union, Text, Tuple
-from qlib.data.dataset.handler import DataHandler
+from qlib.model.meta.dataset import MetaTaskDataset
+from qlib.model.trainer import TrainerR
 from qlib.log import get_module_logger
 from qlib.utils import auto_filter_kwargs, get_date_by_shift, init_instance_by_config
+from qlib.utils.data import deepcopy_basic_type
 from qlib.workflow import R
 from qlib.workflow.task.gen import RollingGen, task_generator
-from joblib import Parallel, delayed
-from qlib.model.meta.dataset import MetaTaskDataset
-from qlib.model.trainer import task_train, TrainerR
-from qlib.data.dataset import DatasetH
+from qlib.workflow.task.utils import TimeAdjuster
 from tqdm.auto import tqdm
-import pandas as pd
-import numpy as np
 
 
 class InternalData:

qlib/contrib/meta/data_selection/model.py

@@ -1,28 +1,26 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from qlib.log import get_module_logger
 import pandas as pd
 import numpy as np
-from qlib.model.meta.task import MetaTask
 import torch
 from torch import nn
 from torch import optim
 from tqdm.auto import tqdm
-import collections
 import copy
-from typing import Union, List, Tuple, Dict
+from typing import Union, List
 
 from ....data.dataset.weight import Reweighter
 from ....model.meta.dataset import MetaTaskDataset
-from ....model.meta.model import MetaModel, MetaTaskModel
+from ....model.meta.model import MetaTaskModel
 from ....workflow import R
-
 from .utils import ICLoss
 from .dataset import MetaDatasetDS
-from qlib.contrib.meta.data_selection.net import PredNet
-from qlib.data.dataset.weight import Reweighter
+
 from qlib.log import get_module_logger
+from qlib.data.dataset.weight import Reweighter
+from qlib.model.meta.task import MetaTask
+from qlib.contrib.meta.data_selection.net import PredNet
 
 logger = get_module_logger("data selection")