network_report_server/network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: pip in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (23.3.2)\n",
      "Requirement already satisfied: setuptools in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (68.0.0)\n",
      "Requirement already satisfied: wheel in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (0.42.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip setuptools wheel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: pip is being invoked by an old script wrapper. This will fail in a future version of pip.\n",
      "Please see https://github.com/pypa/pip/issues/5599 for advice on fixing the underlying issue.\n",
      "To avoid this problem you can invoke Python with '-m pip' instead of running pip directly.\n",
      "Requirement already satisfied: p5py in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (1.0.0)\n",
      "Requirement already satisfied: PEP517 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (0.13.1)\n",
      "Requirement already satisfied: pymongo in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (4.6.1)\n",
      "Requirement already satisfied: pandas in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (1.3.5)\n",
      "Requirement already satisfied: tomli>=1.1.0 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from PEP517) (2.0.1)\n",
      "Requirement already satisfied: importlib_metadata in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from PEP517) (6.7.0)\n",
      "Requirement already satisfied: zipp in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from PEP517) (3.15.0)\n",
      "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from pymongo) (2.3.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /snap/jupyter/6/lib/python3.7/site-packages (from pandas) (2.8.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from pandas) (2023.3.post1)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from pandas) (1.21.6)\n",
      "Requirement already satisfied: six>=1.5 in /snap/jupyter/6/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas) (1.12.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /home/tum/snap/jupyter/common/lib/python3.7/site-packages (from importlib_metadata->PEP517) (4.7.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install p5py PEP517 pymongo pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = MongoClient(\"mongodb://localhost\")\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['admin', 'config', 'local']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.list_database_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "db = client.network_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = db['exfo_api'].find()\n",
    "len(list(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "all_result = list(db['exfo_api'].find({'ids.sla_id': 1350}))\n",
    "all_df = pd.DataFrame(all_result)\n",
    "all_df['ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "AutoReconnect",
     "evalue": "localhost:27017: connection closed (configured timeouts: connectTimeoutMS: 20000.0ms)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/pool.py\u001b[0m in \u001b[0;36mreceive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m   1019\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1020\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mreceive_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_message_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1021\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/network.py\u001b[0m in \u001b[0;36mreceive_message\u001b[0;34m(conn, request_id, max_message_size)\u001b[0m\n\u001b[1;32m    256\u001b[0m     \u001b[0;31m# Ignore the response's request id.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 257\u001b[0;31m     \u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse_to\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_UNPACK_HEADER\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_receive_data_on_socket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeadline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m     \u001b[0;31m# No request_id for exhaust cursor \"getMore\".\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/network.py\u001b[0m in \u001b[0;36m_receive_data_on_socket\u001b[0;34m(conn, length, deadline)\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mchunk_length\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"connection closed\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    349\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: connection closed",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mAutoReconnect\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-a4ba2bdec62f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mr0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'exfo_api'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'test_status_result'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m{\u001b[0m \u001b[0;34m'$exists'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'$ne'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m}\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"created\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mr0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/cursor.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1262\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__empty\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1263\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1264\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__data\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_refresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1265\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopleft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/cursor.py\u001b[0m in \u001b[0;36m_refresh\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1179\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__exhaust\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1180\u001b[0m             )\n\u001b[0;32m-> 1181\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__send_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1182\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__id\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Get More\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1183\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__limit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/cursor.py\u001b[0m in \u001b[0;36m__send_message\u001b[0;34m(self, operation)\u001b[0m\n\u001b[1;32m   1059\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1060\u001b[0m             response = client._run_operation(\n\u001b[0;32m-> 1061\u001b[0;31m                 \u001b[0moperation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unpack_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maddress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__address\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1062\u001b[0m             )\n\u001b[1;32m   1063\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOperationFailure\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/_csot.py\u001b[0m in \u001b[0;36mcsot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    105\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0m_TimeoutContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcsot_wrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_run_operation\u001b[0;34m(self, operation, unpack_res, address)\u001b[0m\n\u001b[1;32m   1397\u001b[0m             \u001b[0moperation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1398\u001b[0m             \u001b[0maddress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1399\u001b[0;31m             \u001b[0mretryable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Query\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1400\u001b[0m         )\n\u001b[1;32m   1401\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_retryable_read\u001b[0;34m(self, func, read_pref, session, address, retryable)\u001b[0m\n\u001b[1;32m   1497\u001b[0m             \u001b[0maddress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1498\u001b[0m             \u001b[0mread_pref\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mread_pref\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1499\u001b[0;31m             \u001b[0mretryable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretryable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1500\u001b[0m         )\n\u001b[1;32m   1501\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/_csot.py\u001b[0m in \u001b[0;36mcsot_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    105\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0m_TimeoutContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcsot_wrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_retry_internal\u001b[0;34m(self, func, session, bulk, is_read, address, read_pref, retryable)\u001b[0m\n\u001b[1;32m   1459\u001b[0m             \u001b[0mread_pref\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mread_pref\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1460\u001b[0m             \u001b[0maddress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1461\u001b[0;31m             \u001b[0mretryable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretryable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1462\u001b[0m         ).run()\n\u001b[1;32m   1463\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2318\u001b[0m                 \u001b[0;31m# if we raise ServerSelectionTimeoutError on the retry\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2319\u001b[0m                 \u001b[0;31m# attempt. Raise the original exception instead.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2320\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_last_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2321\u001b[0m                 \u001b[0;31m# A ServerSelectionTimeoutError error indicates that there may\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2322\u001b[0m                 \u001b[0;31m# be a persistent outage. Attempting to retry in this case will\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_check_last_error\u001b[0;34m(self, check_csot)\u001b[0m\n\u001b[1;32m   2390\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcheck_csot\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mremaining\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mremaining\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2391\u001b[0m                 \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_error\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2392\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2393\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2394\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_server\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mServer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2313\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_last_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_csot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2314\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2315\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_read\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_write\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2316\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mServerSelectionTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2317\u001b[0m                 \u001b[0;31m# The application may think the write was never attempted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   2443\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retrying\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_retryable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2444\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_last_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2445\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_server\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread_pref\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/mongo_client.py\u001b[0m in \u001b[0;36m_cmd\u001b[0;34m(_session, server, conn, read_preference)\u001b[0m\n\u001b[1;32m   1389\u001b[0m             \u001b[0moperation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Reset op in case of retry.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1390\u001b[0m             return server.run_operation(\n\u001b[0;32m-> 1391\u001b[0;31m                 \u001b[0mconn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mread_preference\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event_listeners\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munpack_res\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1392\u001b[0m             )\n\u001b[1;32m   1393\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/helpers.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 322\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    323\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOperationFailure\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    324\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mno_reauth\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/server.py\u001b[0m in \u001b[0;36mrun_operation\u001b[0;34m(self, conn, operation, read_preference, listeners, unpack_res)\u001b[0m\n\u001b[1;32m    146\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m                 \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_doc_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m                 \u001b[0mreply\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreceive_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m             \u001b[0;31m# Unpack and check for command errors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/pool.py\u001b[0m in \u001b[0;36mreceive_message\u001b[0;34m(self, request_id)\u001b[0m\n\u001b[1;32m   1020\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mreceive_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_message_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_connection_failure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_raise_if_not_writable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munacknowledged\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/pool.py\u001b[0m in \u001b[0;36m_raise_connection_failure\u001b[0;34m(self, error)\u001b[0m\n\u001b[1;32m   1169\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mIOError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSSLError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1170\u001b[0m             \u001b[0mdetails\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_timeout_details\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1171\u001b[0;31m             \u001b[0m_raise_connection_failure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddress\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_details\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdetails\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1172\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1173\u001b[0m             \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/tum/snap/jupyter/common/lib/python3.7/site-packages/pymongo/pool.py\u001b[0m in \u001b[0;36m_raise_connection_failure\u001b[0;34m(address, error, msg_prefix, timeout_details)\u001b[0m\n\u001b[1;32m    409\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mNetworkTimeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mAutoReconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    412\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAutoReconnect\u001b[0m: localhost:27017: connection closed (configured timeouts: connectTimeoutMS: 20000.0ms)"
     ]
    }
   ],
   "source": [
    "r0 = list(db['exfo_api'].find({'test_status_result': { '$exists': True, '$ne': [] }}).sort(\"created\", -1).limit(10))\n",
    "r0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r0 = list(db['exfo_api'].find({'test_status_result': { '$exists': True, '$ne': [] }, 'ids':{'sla_name': {'$regex': '*POC*'}}}).sort(\"created\", -1).limit(10))\n",
    "r0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r0 = list(db['exfo_api'].find({'test_status_result': { '$exists': True, '$ne': [] }}).sort(\"created\", -1).limit(10))\n",
    "pd.json_normalize(r0, ['test_status_result','status', 'results_scores'], ['test_status_result','test_instance_id'], errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "666\n",
      "30\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "list_test_status_result = list(db['exfo_api'].find({'test_status_result': { '$exists': True, '$ne': [] }}, {'test_status_result':1, 'ids': 1}).sort(\"created\", -1).limit(30))\n",
    "print(len(list_test_status_result))\n",
    "data = []\n",
    "data2 = []\n",
    "for l in list_test_status_result:\n",
    "    data = data + l['test_status_result']\n",
    "    data2.append(l['ids'])\n",
    "print(len(data))\n",
    "print(len(data2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "666\n"
     ]
    }
   ],
   "source": [
    "df0 = pd.json_normalize(data)\n",
    "df0\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ATP-Expansion BRIX', 'BV110_NMA', 'BV110_KKN', 'BV110_HYI',\n",
       "       'BV110_SNI', 'BV1100_BRK', 'BV110_PTY', 'BV110_CMI', 'BV110_NWS',\n",
       "       'BV110_PLK', 'BV110_RBI'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0['sla_name'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df0['sla_name'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ids = pd.json_normalize(data2)\n",
    "df_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.json_normalize(data, record_path=['status', 'results_scores'], \n",
    "                       meta=['test_instance_id', ['status','time_stamp_display']],  errors='ignore')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3 = pd.merge(df0, df, on='test_instance_id')\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3.drop(['status'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3.query('test_instance_class_id == 1554')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ping_test = df3.query(\"`service.service_name` == 'Ping test'\").drop([\"status\"],axis=1)\n",
    "ping_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ping_test.to_excel(\"ping_test.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ping_test.dtypes\n",
    "ping_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ping_test[\"service.service_name\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3['status'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3[\"value_score\"] = df[[\"raw_value\",\"score\"]].astype(str).agg(\"/\".join, axis=1)\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(df3.query(\"sla_name == 'BV110_RBI'\"))\n",
    "print(df3[\"sla_name\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df4 = df3.query(\"sla_name in ('POC-Expansion-SLA_BRAS_88_Private (VLAN10)','POC-Expansion-SLA_BNG_144_Public (VLAN20)','POC-Expansion-SLA_BNG_144_Private (VLAN10)')\")\n",
    "df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df3['sla_name'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "table = pd.pivot_table(df3, index=['sla_name', 'status.time_stamp_display'], \n",
    "                       columns=['service.service_name',  \"target.target_value\", \"name\"], values=[\"raw_value\",])\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    " \n",
    "# ts stores the time in seconds\n",
    "ts = int(time.time())\n",
    "table.to_excel(f\"table{ts}.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3['status'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = pd.json_normalize(list_test_status_result, record_path=[\"test_status_result\",\"status\"], meta=[\"\"])\n",
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict_list_to_df(df, col):\n",
    "    \"\"\"Return a Pandas dataframe based on a column that contains a list of JSON objects or dictionaries.\n",
    "    Args:\n",
    "        df (Pandas dataframe): The dataframe to be flattened.\n",
    "        col (str): The name of the column that contains the JSON objects or dictionaries.\n",
    "    Returns:\n",
    "        Pandas dataframe: A new dataframe with the JSON objects or dictionaries expanded into columns.\n",
    "    \"\"\"\n",
    "\n",
    "    rows = []\n",
    "    for index, row in df[col].iteritems():\n",
    "        for item in row:\n",
    "            rows.append(item)\n",
    "    df = pd.DataFrame(rows)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = dict_list_to_df(df, \"status\")\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = dict_list_to_df(df1, \"results_scores\")\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install flatten-json\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flatten_json import flatten\n",
    "df['status_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_flat = pd.io.json.json_normalize(json_struct)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "from pandas.io.json import json_normalize\n",
    "\n",
    "def only_dict(d):\n",
    "    '''\n",
    "    Convert json string representation of dictionary to a python dict\n",
    "    '''\n",
    "    return ast.literal_eval(d)\n",
    "\n",
    "def list_of_dicts(ld):\n",
    "    '''\n",
    "    Create a mapping of the tuples formed after \n",
    "    converting json strings of list to a python list   \n",
    "    '''\n",
    "    return dict([(list(d.values())[1], list(d.values())[0]) for d in ast.literal_eval(ld)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = json_normalize(df['status'].apply(only_dict).tolist()).add_prefix('status.')\n",
    "A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.json_normalize(list(df['status']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_temp = df.explode('status')\n",
    "new_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df2 = df.merge(new_temp, left_index=True, right_index=True, suffixes=(\"_test\",\"_status\"))\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.to_excel(\"test.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df.explode('status.results_scores')\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "l1 = []\n",
    "for l in list_test_status_result:\n",
    "    l1 = l1 + l['test_status_result']\n",
    "df = pd.DataFrame(l1)\n",
    "df = pd.concat([df, df[\"service\"].apply(pd.Series)], axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['results_scores']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "df1 = pd.DataFrame(df['status'][0])\n",
    "temp_df = pd.DataFrame.from_dict(np.concatenate(df1['results_scores']).tolist())\n",
    "temp_df[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = df1.merge(temp_df, how='inner', left_index=True, right_index=True)\n",
    "final_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list_test_status_result)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.explode('test_status_result')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "l0 = []\n",
    "for l in list_test_status_result:\n",
    "    for t in l['test_status_result']:\n",
    "        for st in t['status']:\n",
    "            for rs in st['results_scores']:\n",
    "                temp = t['service'] | (t['target'] or {}) | rs\n",
    "                temp['sla_name'] = st['sla_name'] \n",
    "                temp['time_stamp'] = st['time_stamp']\n",
    "                temp['time_stamp_display'] = st['time_stamp_display']\n",
    "                l0.append(temp)\n",
    "l0[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_test_status_result = list(db['exfo_api'].find({}, {'test_status_result':1}))\n",
    "list_test_status_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "\n",
    "temp = []\n",
    "for l in list_test_status_result:\n",
    "    for d in l['test_status_result']:\n",
    "        for s in d['status']:\n",
    "            for rs in s['results_scores']:\n",
    "                t2 = {rs['name']: [rs['raw_value'], rs['score']]}\n",
    "                temp.append({**d, **s, **t2})\n",
    "temp\n",
    "df = pd.json_normalize(temp)\n",
    "#df.set_index(['time_stamp_display'], inplace=True)\n",
    "df['sla_service'] = df['sla_name'] + \"/\" + df['service_name']\n",
    "df['sla_service'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = df[df['sla_service'] == 'BV110_CMI/HTTPS Web Performances']\n",
    "f.set_index(['time_stamp_display'], inplace=True)\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}