Skip to content

Commit

Permalink
fix panic during normalizing the invalid data
Browse files Browse the repository at this point in the history
  • Loading branch information
m3ngyang committed Nov 22, 2023
1 parent 98f569e commit 03368d8
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions scripts/data_collector/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,12 +301,15 @@ def _executor(self, file_path: Path):
na_values={col: symbol_na if col == self._symbol_field_name else default_na for col in columns},
)

df = self._normalize_obj.normalize(df)
if df is not None and not df.empty:
if self._end_date is not None:
_mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
df = df[_mask]
df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
try:
df = self._normalize_obj.normalize(df)
if df is not None and not df.empty:
if self._end_date is not None:
_mask = pd.to_datetime(df[self._date_field_name]) <= pd.Timestamp(self._end_date)
df = df[_mask]
df.to_csv(self._target_dir.joinpath(file_path.name), index=False)
except Exception as e:
logger.error(f"normalize {file_path.name} failed, error: {e}")

def normalize(self):
logger.info("normalize data......")
Expand Down

0 comments on commit 03368d8

Please sign in to comment.