intake_complex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d383673b-c2ed-4249-8dca-209f81d2155f",
   "metadata": {},
   "source": [
    "Describe test ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c1241b53-503d-4edd-8a9d-1d4adebcdf46",
   "metadata": {},
   "outputs": [],
   "source": [
    "import intake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d4e1b333-89a5-4ed9-8025-5604b1d11c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = intake.cat.nci\n",
    "cmip6 = cat.esgf.cmip6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea042e4e-832a-4d09-8808-b292e0e04845",
   "metadata": {},
   "outputs": [],
   "source": [
    "# I changed a bit the original constraints to get less results\n",
    "subset = cmip6.search(require_all_on=['source_id','member_id'], \n",
    "                      experiment_id=['hist-GHG','hist-aer'], \n",
    "                      variable_id=['thetao','tasmin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c07c6b63-0965-400b-b659-f6001ab663ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "runs = subset.df.groupby(['source_id','member_id', 'experiment_id'])\n",
    "len(runs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "610b88e0-ce62-44ef-bb84-189c614b7db6",
   "metadata": {},
   "source": [
    "In the filter we first sort the date_range values so we can pick the frist and last date_range easily. Then we get the starting year (first 4 digits of first date_range) and the last year (split the date_range and match first 4 digits of second date).<br>\n",
    "The filter returns False if they don't match the years we set as start and end: 1850/2020."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b7e4c30-321a-40fc-b0d9-a6b0787ac1e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_range(group, ystart=1850, yend=2020):\n",
    "    \"\"\" Check if dates passed matched start/end years\n",
    "        we could just check that start and end year are the same but we really want\n",
    "        to include also runs that starts earlier or end later\n",
    "    \"\"\"\n",
    "    check = False\n",
    "    drange = group.date_range.sort_values()\n",
    "    dstart = drange.iloc[0][:4]\n",
    "    dend = drange.iloc[-1].split(\"-\")[1][:4]\n",
    "    if int(dstart) <= ystart and int(dend) >= yend:\n",
    "        check = True\n",
    "    if check is False: print(dstart, dend)\n",
    "    return check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3217f7b2-a210-4bec-88a2-a43e10491399",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1850 2015\n",
      "1850 2015\n",
      "1850 2014\n",
      "1850 2015\n"
     ]
    }
   ],
   "source": [
    "dfnew = subset.df.groupby(['source_id','member_id', 'experiment_id']).filter(lambda x: check_range(x) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b585d2d1-ae47-403a-ac12-9032be803789",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>project</th>\n",
       "      <th>activity_id</th>\n",
       "      <th>institution_id</th>\n",
       "      <th>table_id</th>\n",
       "      <th>variable_id</th>\n",
       "      <th>grid_label</th>\n",
       "      <th>date_range</th>\n",
       "      <th>path</th>\n",
       "      <th>version</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>experiment_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ACCESS-CM2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">r1i1p1f1</th>\n",
       "      <th>hist-GHG</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist-aer</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">r2i1p1f1</th>\n",
       "      <th>hist-GHG</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist-aer</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r3i1p1f1</th>\n",
       "      <th>hist-GHG</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">NorESM2-LM</th>\n",
       "      <th>r1i1p1f1</th>\n",
       "      <th>hist-aer</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">r2i1p1f1</th>\n",
       "      <th>hist-GHG</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist-aer</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">r3i1p1f1</th>\n",
       "      <th>hist-GHG</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist-aer</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>94 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    project  activity_id  institution_id  \\\n",
       "source_id  member_id experiment_id                                         \n",
       "ACCESS-CM2 r1i1p1f1  hist-GHG          True         True            True   \n",
       "                     hist-aer          True         True            True   \n",
       "           r2i1p1f1  hist-GHG          True         True            True   \n",
       "                     hist-aer          True         True            True   \n",
       "           r3i1p1f1  hist-GHG          True         True            True   \n",
       "...                                     ...          ...             ...   \n",
       "NorESM2-LM r1i1p1f1  hist-aer          True         True            True   \n",
       "           r2i1p1f1  hist-GHG          True         True            True   \n",
       "                     hist-aer          True         True            True   \n",
       "           r3i1p1f1  hist-GHG          True         True            True   \n",
       "                     hist-aer          True         True            True   \n",
       "\n",
       "                                    table_id  variable_id  grid_label  \\\n",
       "source_id  member_id experiment_id                                      \n",
       "ACCESS-CM2 r1i1p1f1  hist-GHG           True         True        True   \n",
       "                     hist-aer           True         True        True   \n",
       "           r2i1p1f1  hist-GHG           True         True        True   \n",
       "                     hist-aer           True         True        True   \n",
       "           r3i1p1f1  hist-GHG           True         True        True   \n",
       "...                                      ...          ...         ...   \n",
       "NorESM2-LM r1i1p1f1  hist-aer           True         True        True   \n",
       "           r2i1p1f1  hist-GHG           True         True        True   \n",
       "                     hist-aer           True         True        True   \n",
       "           r3i1p1f1  hist-GHG           True         True        True   \n",
       "                     hist-aer           True         True        True   \n",
       "\n",
       "                                    date_range  path  version  \n",
       "source_id  member_id experiment_id                             \n",
       "ACCESS-CM2 r1i1p1f1  hist-GHG             True  True     True  \n",
       "                     hist-aer             True  True     True  \n",
       "           r2i1p1f1  hist-GHG             True  True     True  \n",
       "                     hist-aer             True  True     True  \n",
       "           r3i1p1f1  hist-GHG             True  True     True  \n",
       "...                                        ...   ...      ...  \n",
       "NorESM2-LM r1i1p1f1  hist-aer             True  True     True  \n",
       "           r2i1p1f1  hist-GHG             True  True     True  \n",
       "                     hist-aer             True  True     True  \n",
       "           r3i1p1f1  hist-GHG             True  True     True  \n",
       "                     hist-aer             True  True     True  \n",
       "\n",
       "[94 rows x 9 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfnew.reset_index(drop=True, inplace=True)\n",
    "dfnew.groupby(['source_id','member_id', 'experiment_id']).all()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d99184fb-6a59-4c78-9faa-8137e04b64d0",
   "metadata": {},
   "source": [
    "Let's see how this would work with your smaller example. This model has 3 members and 1 starts from 1930 instead of 1850. We introduced the print statement to test that the filter can detect this correctly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e65eaa4d-c8ed-42cb-a243-3d4469dcb389",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = cmip6.search(variable_id=['thetao'], table_id=['Omon'], grid_label=['gn'], \n",
    "                      source_id=['BCC-CSM2-MR'], experiment_id=['hist-aer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "36a51218-7146-41b8-8307-a4b61db8c013",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1930 2020\n"
     ]
    }
   ],
   "source": [
    "dfnew = subset.df.groupby('member_id').filter(lambda x: check_range(x) )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84a88197-12be-42ab-839c-58efadfaa130",
   "metadata": {},
   "source": [
    "The filter seems to work, by grouping again by member_id we can see only the 2 complete members are left."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e17dc310-799e-40da-b135-84464f55e98e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>project</th>\n",
       "      <th>activity_id</th>\n",
       "      <th>institution_id</th>\n",
       "      <th>source_id</th>\n",
       "      <th>experiment_id</th>\n",
       "      <th>table_id</th>\n",
       "      <th>variable_id</th>\n",
       "      <th>grid_label</th>\n",
       "      <th>date_range</th>\n",
       "      <th>path</th>\n",
       "      <th>version</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>member_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>r1i1p1f1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r2i1p1f1</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           project  activity_id  institution_id  source_id  experiment_id  \\\n",
       "member_id                                                                   \n",
       "r1i1p1f1      True         True            True       True           True   \n",
       "r2i1p1f1      True         True            True       True           True   \n",
       "\n",
       "           table_id  variable_id  grid_label  date_range  path  version  \n",
       "member_id                                                                \n",
       "r1i1p1f1       True         True        True        True  True     True  \n",
       "r2i1p1f1       True         True        True        True  True     True  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfnew.reset_index(drop=True, inplace=True) # reset index\n",
    "dfnew.groupby('member_id').all()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd5efc4b-8ae4-4b8b-ae33-7d60dd9ef204",
   "metadata": {},
   "source": [
    "The problem is that this now a Pandas DataFrame, so we lost the link to the intake catalogue. It is not possible yet to run such a query in intake. However, once we identified which members to remove using the filter we can build their intake key and remove them from the dataset dictionary.<br>\n",
    "To do this more easily we will modify the function to return the dataset key instead of True and False."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b3a27875-faa2-4bc3-83cd-43193c0327d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['project',\n",
       " 'activity_id',\n",
       " 'institution_id',\n",
       " 'source_id',\n",
       " 'experiment_id',\n",
       " 'member_id',\n",
       " 'table_id',\n",
       " 'variable_id',\n",
       " 'grid_label',\n",
       " 'version']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmip6.aggregation_info.groupby_attrs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fe61d6b-dc15-4f24-b9a5-6e6d0fcaf325",
   "metadata": {},
   "source": [
    "Based on the aggregation_info we can build a dataset key using the groupby columns:\n",
    "    'project', 'activity_id', 'institution_id', 'source_id', 'experiment_id', 'member_id', 'table_id', 'variable_id', 'grid_label', 'version'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f25a7236-40ef-485a-8a81-09c66abe4d90",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_range(group, ystart=1850, yend=2020):\n",
    "    \"\"\" Check if dates passed matched start/end years\"\"\"\n",
    "    attrs = cmip6.aggregation_info.groupby_attrs\n",
    "    drange = group.date_range.sort_values()\n",
    "    dstart = drange.iloc[0][:4]\n",
    "    dend = drange.iloc[-1].split(\"-\")[1][:4]    \n",
    "    # build back key\n",
    "    ds_key = \".\".join([group[x].iloc[0] for x in attrs if x in group.columns])\n",
    "    if int(dstart) > ystart or int(dend) < yend:\n",
    "        return ds_key"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd66413a-aece-422b-a701-341628967ccc",
   "metadata": {},
   "source": [
    "Let's test this on our small example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "7d840646-1a4c-458f-8212-3df6d77eda3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_remove = [check_range(x[1]) for x in subset.df.groupby('member_id')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05325dd3-fc31-40a4-837f-c8b813cb11cf",
   "metadata": {},
   "source": [
    "We need to remove the None values returned when the member range was fine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "338564f3-4b97-454d-b77c-300adc2a2bce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CMIP6.DAMIP.BCC.BCC-CSM2-MR.hist-aer.r3i1p1f1.Omon.thetao.gn.v20190516']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_remove = list(filter(None, to_remove))\n",
    "to_remove"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75afe5d5-cba5-4937-b4fe-9ca47f54dd6b",
   "metadata": {},
   "source": [
    "Finally, we can create a dataset_dict and then remove this key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9352f42f-18a9-411e-932f-9845b6fe4697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--> The keys in the returned dictionary of datasets are constructed as follows:\n",
      "\t'project.activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.version'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='3' class='' max='3' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [3/3 00:00&lt;00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds_dict = subset.to_dataset_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "581f6bc0-b556-4765-a168-5b8da9bb77bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "for k in to_remove:\n",
    "    ds_dict.pop(k) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d538fa42-9771-4642-8087-9e492a7f4592",
   "metadata": {},
   "source": [
    "Let's check that the dataset r3i1p1f1 has been removed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfff3e4e-3733-4649-8a38-75195a0fc466",
   "metadata": {},
   "source": [
    "Can I now rerun the subset search but passing only this table instead of the all CMIP6 catalogue table?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0c4c7217-eccb-4c2d-8cca-234a5206f64d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['CMIP6.DAMIP.BCC.BCC-CSM2-MR.hist-aer.r2i1p1f1.Omon.thetao.gn.v20190513', 'CMIP6.DAMIP.BCC.BCC-CSM2-MR.hist-aer.r1i1p1f1.Omon.thetao.gn.v20190516'])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_dict.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b899ac12-3a86-4c84-a772-a483dc732971",
   "metadata": {},
   "source": [
    "Let's finally try to see if this work on a bigger subset, we repeat the first query which returned 156 runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e6a44b6f-82cd-4261-ba10-9005cc941d69",
   "metadata": {},
   "outputs": [],
   "source": [
    "subset = cmip6.search(require_all_on=['source_id','member_id'], experiment_id=['hist-GHG','hist-aer'], variable_id=['thetao','tasmin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d40eb410-61ea-4e1a-a602-38228fdf3f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "to_remove = [check_range(x[1]) for x in subset.df.groupby(['source_id','member_id', 'experiment_id', 'variable_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "4f242949-fee5-4b2b-8602-5b3cd2111695",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "196"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_remove = list(filter(None, to_remove))\n",
    "print(len(to_remove))\n",
    "len(subset.df.groupby(['source_id','member_id', 'experiment_id', 'variable_id']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa7dbd54-0a46-44e1-8746-a59efd962925",
   "metadata": {},
   "source": [
    "There's 9 simulations  out of 196 to remove let's check a few to be sure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e1a81f2-32ae-40e1-aa6b-459c9cb496b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--> The keys in the returned dictionary of datasets are constructed as follows:\n",
      "\t'project.activity_id.institution_id.source_id.experiment_id.member_id.table_id.variable_id.grid_label.version'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      <progress value='6' class='' max='228' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      2.63% [6/228 00:18&lt;11:29]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/g/data/hh5/public/apps/miniconda3/envs/analysis3-22.10/lib/python3.9/site-packages/xarray/conventions.py:523: SerializationWarning: variable 'thetao' has multiple fill values {1e+20, 1e+20}, decoding all values to NaN.\n",
      "  new_vars[k] = decode_cf_variable(\n",
      "/g/data/hh5/public/apps/miniconda3/envs/analysis3-22.10/lib/python3.9/site-packages/xarray/conventions.py:523: SerializationWarning: variable 'thetao' has multiple fill values {1e+20, 1e+20}, decoding all values to NaN.\n",
      "  new_vars[k] = decode_cf_variable(\n",
      "/g/data/hh5/public/apps/miniconda3/envs/analysis3-22.10/lib/python3.9/site-packages/xarray/conventions.py:523: SerializationWarning: variable 'thetao' has multiple fill values {1e+20, 1e+20}, decoding all values to NaN.\n",
      "  new_vars[k] = decode_cf_variable(\n"
     ]
    }
   ],
   "source": [
    "ds_dict = subset.to_dataset_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0203fdcf-73c7-406e-adf2-b52d498be678",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_dict.keys(to_remove[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49abe368-281e-4275-9885-776853e09b77",
   "metadata": {},
   "source": [
    "So far this is sub-optimal as you still have to aggregate the datasets before you can remove the `faulty` ones (hence I used constraints that would return less simulations). What I still need to work out is how to filter out efficiently the `subset` before this step. From the intake-esm docs there's a way to pass a filter excluding a parituclar mdoel, is not clear to me how to exclude a combination, but this is what I will try after.<br>\n",
    "In fact, i didn't even get to fully run the last test :-)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d8a572d-0a32-455d-a49e-a3246896c35b",
   "metadata": {},
   "source": [
    "The current intake-esm release has a bug that breaks our catalogues, we notified them and this has been resolved but they haven't yet produced a new release, so I will try to get the `fixed` version in our conda anyway as then you can do stuff like opening the files in parallel which greatly reduced the running time."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9ef6c36-0a87-4cbb-b554-e7b121f49564",
   "metadata": {},
   "source": [
    "STILL WORKING ON THE STUFF BELOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f38729e5-f915-437a-ab06-6562315b22da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import intake_esm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "415708d2-2009-426b-8359-cbac1221c325",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# I changed a bit the original constraints to get less results\n",
    "newsubset = intake_esm.search.search(df=dfnew, require_all_on=['source_id','member_id'], experiment_id=['hist-GHG','hist-aer'], variable_id=['thetao','tasmin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6119a1c-e99d-485c-9ce0-6005ca27b6f9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0062b24e-332f-4769-b49c-7af2bf21487a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__annotations__',\n",
       " '__call__',\n",
       " '__class__',\n",
       " '__closure__',\n",
       " '__code__',\n",
       " '__defaults__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__get__',\n",
       " '__getattribute__',\n",
       " '__globals__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__kwdefaults__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__name__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__qualname__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(intake_esm.search.search)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:analysis3-unstable]",
   "language": "python",
   "name": "conda-env-analysis3-unstable-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}