fixed fragmentation warnings

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 - Fixed:
   - Add DOROS BPMs to `twiss.dat`.
-  - Pandas `FutureWarning`s
+  - Pandas some `FutureWarning`s and `DeprecationWarning`s
 
 #### 2024-08-14 - v0.15.2 - _fesoubel_, _jdilly_
 

omc3/harpy/frequency.py

@@ -11,6 +11,7 @@
 """
 from collections import OrderedDict
 from numbers import Number
+from typing import List
 
 import numpy as np
 import pandas as pd
@@ -136,7 +137,7 @@ def find_resonances(tunes, nturns, plane, spectra, order_resonances):
     """
     resonance_lines = _get_resonance_lines(order_resonances)
 
-    df = pd.DataFrame(index=spectra["FREQS"].index, columns=OrderedDict())
+    df = pd.DataFrame(index=spectra["FREQS"].index, dtype=pd.Float64Dtype())
     resonances_freqs = _compute_resonances_with_freqs(plane, tunes, resonance_lines)
     if tunes[2] > 0.0:
         resonances_freqs.update(_compute_resonances_with_freqs("Z", tunes, resonance_lines))
@@ -145,10 +146,12 @@ def find_resonances(tunes, nturns, plane, spectra, order_resonances):
         max_coefs, max_freqs = _search_highest_coefs(resonances_freqs[resonance], tolerance,
                                                      spectra["FREQS"], spectra["COEFFS"])
         resstr = _get_resonance_suffix(resonance)
-        df[f"{COL_FREQ}{resstr}"], df[f"{COL_AMP}{resstr}"], df[f"{COL_PHASE}{resstr}"] = _get_freqs_amps_phases(
-            max_freqs, max_coefs, resonances_freqs[resonance])
+        columns=[f"{COL_FREQ}{resstr}", f"{COL_AMP}{resstr}", f"{COL_PHASE}{resstr}"]
+        df_resonance = _get_freqs_amps_phases(max_freqs, max_coefs, resonances_freqs[resonance])
+        df_resonance.columns = columns
+        df.loc[:, columns] = df_resonance
 
-        df[f"{COL_PHASE}{resstr}"] = _realign_phases(df.loc[:, f"{COL_PHASE}{resstr}"].to_numpy(),
+        df.loc[:, f"{COL_PHASE}{resstr}"] = _realign_phases(df.loc[:, f"{COL_PHASE}{resstr}"].to_numpy(),
                                                df.loc[:, f"{COL_FREQ}{resstr}"].to_numpy(), nturns)
 
     return df
@@ -161,25 +164,34 @@ def _get_main_resonances(tunes, spectra, plane, tolerance, df):
         raise ValueError(f"No main {plane} resonances found, "
                          f"try to increase the tolerance or adjust the tunes")
     bad_bpms_by_tune = spectra["COEFFS"].loc[max_coefs == 0.].index
-    df[f"{COL_TUNE}{plane}"], df[f"{COL_AMP}{plane}"], df[f"{COL_MU}{plane}"] = _get_freqs_amps_phases(
-        max_freqs, max_coefs, freq)
+    columns = [f"{COL_TUNE}{plane}", f"{COL_AMP}{plane}", f"{COL_MU}{plane}"]
+    df_main = _get_freqs_amps_phases(max_freqs, max_coefs, freq)
+    df_main.columns = columns
+    df.loc[:, columns] = df_main
     if plane != "Z":
         df = df.loc[df.index.difference(bad_bpms_by_tune)]
     return df, bad_bpms_by_tune
 
 
 def _calculate_natural_tunes(spectra, nattunes, tolerance, plane):
-    df = pd.DataFrame(index=spectra["FREQS"].index, columns=OrderedDict())
+    columns = [f"{COL_NATTUNE}{plane}", f"{COL_NATAMP}{plane}", f"{COL_NATMU}{plane}"]
     x, y, _ = nattunes
     freq = x % 1 if plane == "X" else y % 1
     max_coefs, max_freqs = _search_highest_coefs(freq, tolerance, spectra["FREQS"], spectra["COEFFS"])
-    df[f"{COL_NATTUNE}{plane}"], df[f"{COL_NATAMP}{plane}"], df[f"{COL_NATMU}{plane}"] = _get_freqs_amps_phases(
-        max_freqs, max_coefs, freq)
+    df = _get_freqs_amps_phases(max_freqs, max_coefs, freq)
+    df.columns = columns
     return df
 
 
-def _get_freqs_amps_phases(max_freqs, max_coefs, freq):
-    return max_freqs, np.abs(max_coefs), np.sign(0.5 - freq) * np.angle(max_coefs) / PI2
+def _get_freqs_amps_phases(max_freqs: pd.Series, max_coefs: pd.Series, freq: float) -> pd.DataFrame:
+    return pd.DataFrame(
+        {
+            f"{COL_FREQ}": max_freqs,
+            f"{COL_AMP}": np.abs(max_coefs),
+            f"{COL_PHASE}": np.sign(0.5 - freq) * np.angle(max_coefs) / PI2,
+        },
+        dtype=pd.Float64Dtype(),
+    )
 
 
 def _realign_phases(phase_data, freq_data, nturns):

omc3/harpy/handler.py

@@ -184,29 +184,34 @@ def _get_output_path_without_suffix(output_dir, file_path):
     return join(output_dir, basename(file_path))
 
 
-def _rescale_amps_to_main_line_and_compute_noise(panda, plane):
+def _rescale_amps_to_main_line_and_compute_noise(df, plane):
     """
     TODO    follows non-transpararent convention
     TODO    the consequent analysis has to be changed if removed
     """
-    cols = [col for col in panda.columns.to_numpy() if col.startswith(COL_AMP)]
+    cols = [col for col in df.columns.to_numpy() if col.startswith(COL_AMP)]
     cols.remove(f"{COL_AMP}{plane}")
-    panda.loc[:, cols] = panda.loc[:, cols].div(panda.loc[:, f"{COL_AMP}{plane}"], axis="index")
-    amps = panda.loc[:, f"{COL_AMP}{plane}"].to_numpy()
+    df.loc[:, cols] = df.loc[:, cols].div(df.loc[:, f"{COL_AMP}{plane}"], axis="index")
+    amps = df.loc[:, f"{COL_AMP}{plane}"].to_numpy()
     # Division by two for backwards compatibility with Drive, i.e. the unit is [2mm]
     # TODO  later remove
-    panda[f"{COL_AMP}{plane}"] = panda.loc[:, f"{COL_AMP}{plane}"].to_numpy() / 2
-    if f"{COL_NATAMP}{plane}" in panda.columns:
-        panda[f"{COL_NATAMP}{plane}"] = panda.loc[:, f"{COL_NATAMP}{plane}"].to_numpy() / 2
-
-    if np.max(panda.loc[:, 'NOISE'].to_numpy()) == 0.0:
-        return panda  # Do not calculated errors when no noise was calculated
-    noise_scaled = panda.loc[:, 'NOISE'] / amps
-    panda.loc[:, "NOISE_SCALED"] = noise_scaled
-    panda.loc[:, f"{COL_ERR}{COL_AMP}{plane}"] = panda.loc[:, 'NOISE']
-    if f"{COL_NATTUNE}{plane}" in panda.columns:
-        panda.loc[:, f"{COL_ERR}{COL_NATAMP}{plane}"] = panda.loc[:, 'NOISE']
-    for col in cols:
-        this_amp = panda.loc[:, col]
-        panda.loc[:, f"{COL_ERR}{col}"] = noise_scaled * np.sqrt(1 + np.square(this_amp))
-    return panda
+    df[f"{COL_AMP}{plane}"] = df.loc[:, f"{COL_AMP}{plane}"].to_numpy() / 2
+    if f"{COL_NATAMP}{plane}" in df.columns:
+        df[f"{COL_NATAMP}{plane}"] = df.loc[:, f"{COL_NATAMP}{plane}"].to_numpy() / 2
+
+    if np.max(df.loc[:, 'NOISE'].to_numpy()) == 0.0:
+        return df  # Do not calculated errors when no noise was calculated
+    noise_scaled = df.loc[:, 'NOISE'] / amps
+    df.loc[:, "NOISE_SCALED"] = noise_scaled
+    df.loc[:, f"{COL_ERR}{COL_AMP}{plane}"] = df.loc[:, 'NOISE']
+    if f"{COL_NATTUNE}{plane}" in df.columns:
+        df.loc[:, f"{COL_ERR}{COL_NATAMP}{plane}"] = df.loc[:, 'NOISE']
+
+    # create temporary error-dataframe to speed up the DataFrame building
+    df_amp = pd.DataFrame(
+        data={f"{COL_ERR}{col}": noise_scaled * np.sqrt(1 + np.square(df.loc[:, col])) for col in cols},
+        index=df.index, 
+        dtype=pd.Float64Dtype()
+    )
+    df.loc[:, df_amp.columns] = df_amp
+    return df

omc3/tune_analysis/bbq_tools.py

@@ -144,20 +144,18 @@ def _get_interpolated_moving_average(data_series: pd.Series, clean_mask: Union[p
 
     try:
         # 'interpolate' fills nan based on index/values of neighbours
-        data = data.interpolate("index").fillna(method="bfill").fillna(method="ffill")
+        data = data.interpolate("index").bfill().ffill()
     except TypeError as e:
         raise TypeError("Interpolation failed. "
                         "Usually due to a dtype format that is not properly recognized.") from e
 
     shift = -int((length-1)/2)  # Shift average to middle value
 
     # calculate mean and fill NaNs at the ends
-    data_mav = data.rolling(length).mean().shift(shift).fillna(
-        method="bfill").fillna(method="ffill")
+    data_mav = data.rolling(length).mean().shift(shift).bfill().ffill()
 
     # calculate deviation to the moving average and fill NaNs at the ends
-    std_mav = np.sqrt(((data-data_mav)**2).rolling(length).mean().shift(shift).fillna(
-        method="bfill").fillna(method="ffill"))
+    std_mav = np.sqrt(((data-data_mav)**2).rolling(length).mean().shift(shift).bfill().ffill())
     err_mav = std_mav / np.sqrt(length)
 
     if is_datetime_index:

pyproject.toml

@@ -109,5 +109,5 @@ markers = [
     "cern_network: tests that require access to afs or the technical network",
 ]
 # Helpful for pytest-debugging (leave commented out on commit):
-# log_cli=true
-# log_level=DEBUG
+#log_cli = true
+#log_cli_level = "DEBUG"

tests/unit/test_model_creator.py

@@ -10,7 +10,6 @@
 from omc3.model.manager import get_accelerator
 from omc3.model.model_creators.lhc_model_creator import LhcBestKnowledgeCreator, LhcModelCreator
 from omc3.model_creator import create_instance_and_model
-from omc3.model.model_creators.lhc_model_creator import LhcModelCreator
 from omc3.optics_measurements.constants import NAME
 
 INPUTS = Path(__file__).parent.parent / "inputs"

tests/unit/test_rdt_magnet_order.py

@@ -1,18 +1,10 @@
 import os
-import random
-import string
-import tempfile
-import itertools
 
-import numpy as np
-import pandas as pd
 import pytest
 import tfs
 
 from pathlib import Path
 
-import turn_by_turn as tbt
-from omc3.definitions.constants import PLANES
 from omc3.hole_in_one import hole_in_one_entrypoint
 
 INPUTS = Path(__file__).parent.parent / "inputs"