4-StudyCase-Gorongosa_Mozambique/Treat Data tas NOAA Station and pr meteorological station Gorongosa.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "971d6a54",
   "metadata": {},
   "source": [
    "This notebook contains the treatment of the data from the NOAA stations. Some of the data were absurd for temperature (temperature above the highest registered temperature 57 celsius degrees), so data above 57 degrees were removed.\n",
    "\n",
    "Concerning precipitation data from Gorongosa, a column to count the missing data needed to be added to the dataset, to plot the compare_3_lines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d255dfad",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import os.path\n",
    "\n",
    "import sys \n",
    "sys.path.append(\"../0-Functions\")\n",
    "\n",
    "from Functions_ImportData import import_treat_obs_NOAA\n",
    "from Functions_ImportData import treat_NOAA_data\n",
    "from Functions_ImportData import count_na_in_df_NOAA\n",
    "from Functions_Indicators import str_month\n",
    "from Functions_Indicators import add_year_month_season"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "40688349",
   "metadata": {},
   "outputs": [],
   "source": [
    "daily_sum_obs_from_NOAA = import_treat_obs_NOAA()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bb989f9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>PRCP</th>\n",
       "      <th>PRCP_ATTRIBUTES</th>\n",
       "      <th>TAVG</th>\n",
       "      <th>TAVG_ATTRIBUTES</th>\n",
       "      <th>TMAX</th>\n",
       "      <th>TMAX_ATTRIBUTES</th>\n",
       "      <th>TMIN</th>\n",
       "      <th>TMIN_ATTRIBUTES</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-04-05</td>\n",
       "      <td>2.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>23.8</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>29.0</td>\n",
       "      <td>,D,S</td>\n",
       "      <td>20.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-06-17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>18.2</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>27.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>11.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-06-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.0</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-07-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>21.5</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-07-03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>20.5</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161770</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2020-12-27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.8</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>32.3</td>\n",
       "      <td>,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161771</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2020-12-28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.4</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161772</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2020-12-29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.6</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>26.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161773</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2020-12-30</td>\n",
       "      <td>24.9</td>\n",
       "      <td>,,S</td>\n",
       "      <td>28.3</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>31.2</td>\n",
       "      <td>,,S</td>\n",
       "      <td>25.7</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161774</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>5.1</td>\n",
       "      <td>,,S</td>\n",
       "      <td>28.1</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "      <td>,,S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>161775 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            STATION           NAME  LATITUDE  LONGITUDE  ELEVATION  \\\n",
       "0       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "1       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "2       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "3       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "4       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "...             ...            ...       ...        ...        ...   \n",
       "161770  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "161771  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "161772  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "161773  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "161774  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "\n",
       "              DATE  PRCP PRCP_ATTRIBUTES  TAVG TAVG_ATTRIBUTES  TMAX  \\\n",
       "0       1974-04-05   2.0             ,,S  23.8            H,,S  29.0   \n",
       "1       1974-06-17   0.0             ,,S  18.2            H,,S  27.0   \n",
       "2       1974-06-23   NaN             NaN  21.0            H,,S   NaN   \n",
       "3       1974-07-02   0.0             ,,S  21.5            H,,S   NaN   \n",
       "4       1974-07-03   0.0             ,,S  20.5            H,,S   NaN   \n",
       "...            ...   ...             ...   ...             ...   ...   \n",
       "161770  2020-12-27   NaN             NaN  28.8            H,,S  32.3   \n",
       "161771  2020-12-28   NaN             NaN  29.4            H,,S   NaN   \n",
       "161772  2020-12-29   NaN             NaN  29.6            H,,S   NaN   \n",
       "161773  2020-12-30  24.9             ,,S  28.3            H,,S  31.2   \n",
       "161774  2020-12-31   5.1             ,,S  28.1            H,,S   NaN   \n",
       "\n",
       "       TMAX_ATTRIBUTES  TMIN TMIN_ATTRIBUTES  \n",
       "0                 ,D,S  20.0             ,,S  \n",
       "1                  ,,S  11.0             ,,S  \n",
       "2                  NaN   NaN             NaN  \n",
       "3                  NaN  16.0             ,,S  \n",
       "4                  NaN  16.0             ,,S  \n",
       "...                ...   ...             ...  \n",
       "161770             ,,S   NaN             NaN  \n",
       "161771             NaN  25.0             ,,S  \n",
       "161772             NaN  26.0             ,,S  \n",
       "161773             ,,S  25.7             ,,S  \n",
       "161774             NaN  25.0             ,,S  \n",
       "\n",
       "[161775 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "daily_sum_obs_from_NOAA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "918ceba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# take out impossible observation data\n",
    "\n",
    "## We can see for some maximum temperature observation data that some data points are completely impossible \n",
    "#  (100 for minimum or maximum temperature at Chimoio and Beira sometimes)\n",
    "#daily_sum_obs_from_NOAA=daily_sum_obs_from_NOAA[daily_sum_obs_from_NOAA['TMAX']<80]\n",
    "#daily_sum_obs_from_NOAA=daily_sum_obs_from_NOAA[daily_sum_obs_from_NOAA['TMIN']<80]\n",
    "#daily_sum_obs_from_NOAA=daily_sum_obs_from_NOAA[daily_sum_obs_from_NOAA['TAVG']<80]\n",
    "\n",
    "daily_sum_obs_from_NOAA.loc[daily_sum_obs_from_NOAA['TMAX']>57,'TMAX'] = np.nan\n",
    "daily_sum_obs_from_NOAA.loc[daily_sum_obs_from_NOAA['TMIN']>57,'TMIN'] = np.nan\n",
    "daily_sum_obs_from_NOAA.loc[daily_sum_obs_from_NOAA['TAVG']>57,'TAVG'] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6dfe2afe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:59: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING']=0\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING'][df[name_col].isna()]=1\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:59: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING']=0\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING'][df[name_col].isna()]=1\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:59: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING']=0\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING'][df[name_col].isna()]=1\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:59: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING']=0\n",
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING'][df[name_col].isna()]=1\n"
     ]
    }
   ],
   "source": [
    "daily_sum_obs_from_NOAA = treat_NOAA_data(daily_sum_obs_from_NOAA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9ce3968a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>STATION</th>\n",
       "      <th>NAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ELEVATION</th>\n",
       "      <th>DATE</th>\n",
       "      <th>PRCP</th>\n",
       "      <th>PRCP_ATTRIBUTES</th>\n",
       "      <th>TAVG</th>\n",
       "      <th>TAVG_ATTRIBUTES</th>\n",
       "      <th>...</th>\n",
       "      <th>TMAX_ATTRIBUTES</th>\n",
       "      <th>TMIN</th>\n",
       "      <th>TMIN_ATTRIBUTES</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Season</th>\n",
       "      <th>PRCP MISSING</th>\n",
       "      <th>TAVG MISSING</th>\n",
       "      <th>TMAX MISSING</th>\n",
       "      <th>TMIN MISSING</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-04-05</td>\n",
       "      <td>2.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>23.8</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>,D,S</td>\n",
       "      <td>20.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>1974</td>\n",
       "      <td>Apr</td>\n",
       "      <td>Dry</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-06-17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>18.2</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>,,S</td>\n",
       "      <td>11.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>1974</td>\n",
       "      <td>Jun</td>\n",
       "      <td>Dry</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-06-23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21.0</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1974</td>\n",
       "      <td>Jun</td>\n",
       "      <td>Dry</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-07-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>21.5</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>1974</td>\n",
       "      <td>Jul</td>\n",
       "      <td>Dry</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MZM00067223</td>\n",
       "      <td>MONTEPUEZ, MZ</td>\n",
       "      <td>-13.133</td>\n",
       "      <td>39.033</td>\n",
       "      <td>535.0</td>\n",
       "      <td>1974-07-03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>20.5</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.0</td>\n",
       "      <td>,,S</td>\n",
       "      <td>1974</td>\n",
       "      <td>Jul</td>\n",
       "      <td>Dry</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159722</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2014-12-27</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.8</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159723</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2014-12-28</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.0</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159724</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2014-12-29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.0</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159725</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2014-12-30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.6</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159726</th>\n",
       "      <td>MZ000067297</td>\n",
       "      <td>BEIRA, MZ</td>\n",
       "      <td>-19.800</td>\n",
       "      <td>34.900</td>\n",
       "      <td>16.0</td>\n",
       "      <td>2014-12-31</td>\n",
       "      <td>4.1</td>\n",
       "      <td>,,S</td>\n",
       "      <td>27.2</td>\n",
       "      <td>H,,S</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.6</td>\n",
       "      <td>,,S</td>\n",
       "      <td>2014</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137726 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            STATION           NAME  LATITUDE  LONGITUDE  ELEVATION  \\\n",
       "0       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "1       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "2       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "3       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "4       MZM00067223  MONTEPUEZ, MZ   -13.133     39.033      535.0   \n",
       "...             ...            ...       ...        ...        ...   \n",
       "159722  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "159723  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "159724  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "159725  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "159726  MZ000067297      BEIRA, MZ   -19.800     34.900       16.0   \n",
       "\n",
       "              DATE  PRCP PRCP_ATTRIBUTES  TAVG TAVG_ATTRIBUTES  ...  \\\n",
       "0       1974-04-05   2.0             ,,S  23.8            H,,S  ...   \n",
       "1       1974-06-17   0.0             ,,S  18.2            H,,S  ...   \n",
       "2       1974-06-23   NaN             NaN  21.0            H,,S  ...   \n",
       "3       1974-07-02   0.0             ,,S  21.5            H,,S  ...   \n",
       "4       1974-07-03   0.0             ,,S  20.5            H,,S  ...   \n",
       "...            ...   ...             ...   ...             ...  ...   \n",
       "159722  2014-12-27   NaN             NaN  30.8            H,,S  ...   \n",
       "159723  2014-12-28   NaN             NaN  30.0            H,,S  ...   \n",
       "159724  2014-12-29   NaN             NaN  28.0            H,,S  ...   \n",
       "159725  2014-12-30   NaN             NaN  25.6            H,,S  ...   \n",
       "159726  2014-12-31   4.1             ,,S  27.2            H,,S  ...   \n",
       "\n",
       "        TMAX_ATTRIBUTES  TMIN  TMIN_ATTRIBUTES  Year Month Season  \\\n",
       "0                  ,D,S  20.0              ,,S  1974   Apr    Dry   \n",
       "1                   ,,S  11.0              ,,S  1974   Jun    Dry   \n",
       "2                   NaN   NaN              NaN  1974   Jun    Dry   \n",
       "3                   NaN  16.0              ,,S  1974   Jul    Dry   \n",
       "4                   NaN  16.0              ,,S  1974   Jul    Dry   \n",
       "...                 ...   ...              ...   ...   ...    ...   \n",
       "159722              NaN   NaN              NaN  2014   Dec  Humid   \n",
       "159723              NaN   NaN              NaN  2014   Dec  Humid   \n",
       "159724              NaN   NaN              NaN  2014   Dec  Humid   \n",
       "159725              NaN   NaN              NaN  2014   Dec  Humid   \n",
       "159726              NaN  23.6              ,,S  2014   Dec  Humid   \n",
       "\n",
       "       PRCP MISSING  TAVG MISSING  TMAX MISSING  TMIN MISSING  \n",
       "0                 0             0             0             0  \n",
       "1                 0             0             0             0  \n",
       "2                 1             0             1             1  \n",
       "3                 0             0             1             0  \n",
       "4                 0             0             1             0  \n",
       "...             ...           ...           ...           ...  \n",
       "159722            1             0             1             1  \n",
       "159723            1             0             1             1  \n",
       "159724            1             0             1             1  \n",
       "159725            1             0             1             1  \n",
       "159726            0             0             1             0  \n",
       "\n",
       "[137726 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "daily_sum_obs_from_NOAA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4701c5f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_path = r'C:\\Users\\CLMRX\\COWI\\A248363 - Climate analysis - Documents\\General\\CRVA_tool\\Master_thesis\\Project\\3 - Implementation\\1 - Data\\1-BC\\NOAA-ClimateDataOnline'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a87f2272",
   "metadata": {},
   "outputs": [],
   "source": [
    "name_file = 'filtred_NOAA_obs_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "df1ec686",
   "metadata": {},
   "outputs": [],
   "source": [
    "daily_sum_obs_from_NOAA.to_csv(os.path.join(out_path,name_file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a3cc648",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8b9020b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pr from meteorological station gorongosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1796d9c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>pr</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Season</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1980-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1980</td>\n",
       "      <td>Jan</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1980-01-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1980</td>\n",
       "      <td>Jan</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1980-01-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1980</td>\n",
       "      <td>Jan</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1980-01-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1980</td>\n",
       "      <td>Jan</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1980-01-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1980</td>\n",
       "      <td>Jan</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14971</th>\n",
       "      <td>2020-12-27</td>\n",
       "      <td>2.7</td>\n",
       "      <td>2020</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14972</th>\n",
       "      <td>2020-12-28</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14973</th>\n",
       "      <td>2020-12-29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14974</th>\n",
       "      <td>2020-12-30</td>\n",
       "      <td>138.4</td>\n",
       "      <td>2020</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14975</th>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>80.4</td>\n",
       "      <td>2020</td>\n",
       "      <td>Dec</td>\n",
       "      <td>Humid</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14976 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             time     pr  Year Month Season\n",
       "0      1980-01-01    NaN  1980   Jan  Humid\n",
       "1      1980-01-02    NaN  1980   Jan  Humid\n",
       "2      1980-01-03    NaN  1980   Jan  Humid\n",
       "3      1980-01-04    NaN  1980   Jan  Humid\n",
       "4      1980-01-05    NaN  1980   Jan  Humid\n",
       "...           ...    ...   ...   ...    ...\n",
       "14971  2020-12-27    2.7  2020   Dec  Humid\n",
       "14972  2020-12-28    0.0  2020   Dec  Humid\n",
       "14973  2020-12-29    0.0  2020   Dec  Humid\n",
       "14974  2020-12-30  138.4  2020   Dec  Humid\n",
       "14975  2020-12-31   80.4  2020   Dec  Humid\n",
       "\n",
       "[14976 rows x 5 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = r'C:\\Users\\CLMRX\\COWI\\A248363 - Climate analysis - Documents\\General\\CRVA_tool\\Master_thesis\\Project\\3 - Implementation\\1 - Data\\1-BC\\DirecltyfromMoz\\Precipitation_Gorongosa_reformat.csv'\n",
    "pr_obs_gorongosa_from_gorongosa = pd.read_csv(path)\n",
    "pr_obs_gorongosa_from_gorongosa=add_year_month_season(pr_obs_gorongosa_from_gorongosa,'time')\n",
    "pr_obs_gorongosa_from_gorongosa['pr'][pr_obs_gorongosa_from_gorongosa['pr']=='s/i'] = pr_obs_gorongosa_from_gorongosa['pr'].iloc[0]\n",
    "pr_obs_gorongosa_from_gorongosa['pr'] = pr_obs_gorongosa_from_gorongosa['pr'].astype(float)\n",
    "pr_obs_gorongosa_from_gorongosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4f2cc174",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CLMRX\\OneDrive - COWI\\Documents\\GitHub\\CRVA_tool\\Functions_ImportData.py:60: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df[name_col+' MISSING'][df[name_col].isna()]=1\n"
     ]
    }
   ],
   "source": [
    "# this function 'count_na_in_df_NOAA' count the missing values in the columns named name_col in the dataframe df\n",
    "pr_obs_gorongosa_from_gorongosa_treated=count_na_in_df_NOAA(pr_obs_gorongosa_from_gorongosa,'pr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5a32d2c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "pr_obs_gorongosa_from_gorongosa_treated.to_csv(r'C:\\Users\\CLMRX\\COWI\\A248363 - Climate analysis - Documents\\General\\CRVA_tool\\Master_thesis\\Project\\3 - Implementation\\1 - Data\\1-BC\\DirecltyfromMoz\\Precipitation_Gorongosa_reformat_ready_to_use.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f62880ff",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}