merge

.python-version

+3.10.8

app/backend/templates/backend/gen_report.html

+{% extends "base_raw.html" %}
+{% load backend_tags %}
+{% block content %}
+<h2>Gen Report</h2>
+<a href="#test_result" class='btn btn-link'>Test Result</a>
+<a href="#mikrotik" class='btn btn-link'>Mikrotik</a>
+<form class='my-2'>
+
+{% for sla in  slas %}
+<label class='me-3'>
+  <input type='checkbox' name='sla_name' value='{{ sla }}' {% if sla in request.GET.sla_name %}checked{% endif %}/>{{ sla }}
+</label>
+{% endfor %}
+<input type='submit' name='filter' Value='Filter' class='btn btn-primary ms-3' />
+<a class='btn btn-danger ms-3' href="{% url 'backend:dump_fixed_results' %}?redir={{ request.path_info }}">Dump API</a>
+<input type='submit' name='genReport' value='Gen Report' class='btn btn-info' />
+{% if report_link %}
+  <a href="{{ report_link }}" class='btn btn-link'>Download</a>
+{% endif %}
+</form>
+<h1 id='test_result'>Test Result</h1>
+<div class='table-responsive'>
+  {{ tbl|safe }} 
+</div>
+<h1 id='mikrotik'>Mikrotik</h1>
+<h5 class='text-primary'>IP/ROUTE</h5>
+{% for ip in mk_ips %}
+<h4>{{ ip.gateway }}</h4>
+  <div class='d-flex flex-row justify-content-start  flex-wrap align-content-stretch mb-5'>
+    {% for k,v in ip.items %}
+      <div class='border p-3 fw-bolder'>{{ k }}</div><div class='border p-3'>{{ v }}</div>
+    {% endfor %}
+  </div>
+  
+{% endfor %}
+
+<h5 class='text-primary'>IP/Address</h5>
+<!-- 
+<pre>
+{{ mk_address | pprint }}
+</pre> -->
+{% for ip in mk_address %}
+<h4>{{ ip.interface }}</h4>
+  <div class='d-flex flex-row justify-content-start  flex-wrap align-content-stretch mb-5'>
+    {% for k,v in ip.items %}
+      <div class='border p-3 fw-bolder'>{{ k }}</div><div class='border p-3'>{{ v }}</div>
+    {% endfor %}
+  </div>
+  
+{% endfor %}
+<style>
+th {
+  text-align:center;
+}
+</style>
+{% endblock %}

app/backend/urls.py

     path('reports/', views.reports, name='reports'),
     path('service_status/', views.service_status, name='service_status'),
     path('dump_api/', views.dump_api, name='dump_api'),
+    path('dump_fixed_results/', views.dump_fixed_results, name='dump_fixed_results'),
     path('print_table/', views.print_table, name='print_table'),
+    path('gen_report/', views.gen_report, name='gen_report'), 
 ]

app/backend/views.py

-from django.shortcuts import render
+from django.shortcuts import render, redirect
 from backend.mongodb import db
 from exfo.lib import Exfo, Mikrotik
 from pprint import pprint
 from django.http import JsonResponse, HttpResponse
 from datetime import datetime
 from celery import shared_task
+from django.contrib import messages
 
 exfo = Exfo("administrator", "exf0w0rxC@t4dm!n")
 exfo.login()
 
 
     
+def dump_fixed_results(request):
+    from bson.json_util import dumps
+    from bson.json_util import loads
+    col  = db['fixed_results']
+    col.delete_many({})
+    tt = exfo.call_remote_api("/API/REST/Test/v1/TypeByName?list_all=false&size=0")
+    pprint(tt)
+    tt_json = tt.json()
+    temp = tt_json['result']
+    for t in temp:
+        tn = t['ids']['test_type_name']
+        r = exfo.call_remote_api('/API/REST/Test/v1/FixedResults/'+tn+'?sections=all&size=20&sort=verifier_id asc,time_stamp desc&time_range={"start":"1 weeks  ago","end":"now"}').json()
+        #pprint(r)
+        try:
+            r0 = r['result']
+            if len(r0) > 0:
+                col.insert_many(r0)
+        except Exception as e:
+            pprint(e)
+
+    # pprint(temp)
+    #col.insert_many(temp)
+    results = col.find({})
+    data = dumps(list(results), indent=4)
+    data = {'msg': 'done'}
+    redir = request.GET.get('redir', None)
+    if redir:
+        messages.success(request, 'Dump Fixed Results')
+        return redirect(redir)
+    # return HttpResponse(data, content_type='application/json')
+    return JsonResponse(data)
+
+import humanize
+
+def con_human(r):
+    try:
+        '''
+        if 'time' in r['header'].lower():
+            return r['results']
+        '''
+        x = r['results']
+        n = int(x)
+        if n > 1000:
+            return humanize.naturalsize(n,gnu=True)
+        else:
+            return n
+    except:
+        return r['results']
+def gen_report_notebook():
+    from datetime import datetime
+    import pandas as pd
+
+    c = db['fixed_results'].find()
+    data = []
+    for i in c:
+        #pprint(i['header'])
+        res = dict(zip(i['header'], i['results']))
+        d = {'header': i['header'], 'results': i['results'], 'output': res, 'params': i['parameters'], 'ts': i['ids']['time_stamp'], 
+             'dt': datetime.fromtimestamp(int(i['ids']['time_stamp'])/1000000000), 'test_type_name': i['ids']['fixed_results_url']}
+        d.update(i['names'])
+        #res.update(i['ids'])
+        #res.update(i[])
+        #pprint(d)
+        data.append(d)
+    df0 = pd.DataFrame(data)
+    df0 = df0.explode(["header", "results"])
+    df1 = df0[["header", "results", "dt", "sla_name", "test_display_name"]]
+    df1 = df1.query('sla_name == sla_name')
+    #pprint(df1['sla_name'].unique())
+    
+    #df1['results_text'] = df1['results'].apply(con_human)
+    df1['results_text'] = df1.apply(con_human, axis=1)
+    table = df1.pivot(index=['sla_name', 'dt'],columns=['test_display_name','header'], values='results_text').sort_values(by=['sla_name', 'dt'], ascending=[True, False])
+    table = table.dropna(how='all', axis=0)
+    return (table,df1)
+
+def gen_report(request):
+    pprint("report notebook ...")
+    table,df = gen_report_notebook()
+    slas = list(df['sla_name'].unique())
+    sla_name = request.GET.getlist('sla_name')
+    
+    pprint("--- sla_name ---")
+    pprint(sla_name)
+    if len(sla_name) > 0:
+        sla_filter = ", ".join(f"'{w}'" for w in sla_name)
+        table = table.query(f"sla_name in ({sla_filter})")
+    gen_report = request.GET.get('genReport', None)
+    
+    report_link = None
+    if gen_report:
+        import time
+        ts = int(time.time())
+        fn = f"report_{ts}.xlsx"
+        table.to_excel(f'/code/media/{fn}')
+        report_link = f'/media/{fn}'
+
+    try:
+        mk_ips = mkt.call_remote("ip/route") 
+        mk_address = mkt.call_remote("ip/address") 
+    except:
+        mk_ips = []
+        mk_address = []
+    return render(request, 'backend/gen_report.html', {'tbl': table.to_html(\
+        classes=["table", "table-striped", "table-bordered", "align-middle"],\
+        table_id="report_tbl"), 'slas': slas, 'report_link': report_link, 'mk_ips': mk_ips, 'mk_address': mk_address})

app/exfo/lib.py

         url = self.BASE_URL + "/Login"
         response = self.session.post(url, headers=headers, data=payload, verify=False)
        
-        pprint(self.session.cookies.get_dict())
-        pprint(response.text)
-        pprint(response.cookies)
+        # pprint(self.session.cookies.get_dict())
+        # pprint(response.text)
+        # pprint(response.cookies)
         #self.cookies = response.cookies
     def list_api(self):
         pprint("---- list_api ---")
         }
         url = self.BASE_URL + "/"
         response = self.session.get(url, headers=headers, data=payload, verify=False)
-        pprint(response.json())
+        # pprint(response.json())
         return response
 
     def call_api(self, service, payload={}):

app/network_report/settings.py

 
 
 CELERY_BEAT_SCHEDULE = {
-    "dumpapi": {
-        "task": "backend.views.dump_api_task",
-        #"schedule": crontab(minute="*/1"),
-        # "schedule": crontab(minute="*/30"),
-        # "schedule": crontab(hour="17", minute="15"),
-        "schedule": crontab(minute="*/3", hour="17-21"),
-    },
+    # "dumpapi": {
+        # "task": "backend.views.dump_api_task",
+        # #"schedule": crontab(minute="*/1"),
+        # # "schedule": crontab(minute="*/30"),
+        # # "schedule": crontab(hour="17", minute="15"),
+        # "schedule": crontab(minute="*/3", hour="17-21"),
+    # },
 }

app/templates/_messages.html

+{% if messages %}
+    {% for message in messages %}
+    <div{% if message.tags %} style='z-index:2000' class="alert alert-{{ message.tags }}  text-center h5 position-absolute top-50 start-50 translate-middle fs-6" {% endif %}>{{ message }}</div>
+    {% endfor %}
+{% endif %}
+

app/templates/base_raw.html

+{% load static %}
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="">
+    <meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
+    <meta name="generator" content="Hugo 0.84.0">
+    <title>Dashboard Template · Bootstrap v5.0</title>
+
+    <link rel="canonical" href="https://getbootstrap.com/docs/5.0/examples/dashboard/">
+
+    
+
+    <!-- Bootstrap core CSS -->
+    <link href="{% static "bootstrap/dist/css/bootstrap.css" %}" rel="stylesheet">
+
+    <!-- Favicons -->
+<meta name="theme-color" content="#7952b3">
+
+
+    <style>
+      .bd-placeholder-img {
+        font-size: 1.125rem;
+        text-anchor: middle;
+        -webkit-user-select: none;
+        -moz-user-select: none;
+        user-select: none;
+      }
+
+      @media (min-width: 768px) {
+        .bd-placeholder-img-lg {
+          font-size: 3.5rem;
+        }
+      }
+    </style>
+
+    
+    <!-- Custom styles for this template -->
+    <link href="{% static "css/dashboard.css" %}" rel="stylesheet">
+  </head>
+  <body>
+    
+<header class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow">
+  <a class="navbar-brand col-md-3 col-lg-2 me-0 px-3" href="#">Company name</a>
+  <button class="navbar-toggler position-absolute d-md-none collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation">
+    <span class="navbar-toggler-icon"></span>
+  </button>
+  <input class="form-control form-control-dark w-100" type="text" placeholder="Search" aria-label="Search">
+  <div class="navbar-nav">
+    <div class="nav-item text-nowrap">
+      <a class="nav-link px-3" href="#">Sign out</a>
+    </div>
+  </div>
+</header>
+
+<div class="container-fluid">
+  <div class="row">
+
+    <main class="col-md-12 ms-sm-auto col-lg-12 px-md-4 py-3">
+      {% include "_messages.html" %}
+      {% block content %}
+        <h1>Content is Here</h1>
+      {% endblock %}
+    </main>
+  </div>
+</div>
+
+
+    <script src="{% static "bootstrap/dist/js/bootstrap.bundle.min.js" %}"></script>
+
+      <script src="https://cdn.jsdelivr.net/npm/feather-icons@4.28.0/dist/feather.min.js" integrity="sha384-uO3SXW5IuS1ZpFPKugNNWqTZRRglnUJK6UAZ/gxOX80nxEkN9NcGZTftn6RzhGWE" crossorigin="anonymous"></script>
+      <script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js" integrity="sha384-zNy6FEbO50N+Cg5wap8IKA4M/ZnLJgzc6w2NqACZaK0u0FXfOWRRJOnQtpZun8ha" crossorigin="anonymous"></script>
+      <script src="{% static "alpinejs/dist/cdn.js" %}"></script>
+      <script src="{% static "js/dashboard.js" %}"></script>
+  </body>
+</html>

network.ipynb

   {
    "cell_type": "code",
    "execution_count": 37,
+   "id": "93bbfffb",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": 38,
+   "id": "96e5adc4",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": 39,
+   "id": "ff046a48",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": 40,
+   "id": "220562c3",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": 41,
+   "id": "46daba6c",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": 42,
+   "id": "ed775449",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": 43,
+   "id": "1996a9ee",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": 44,
+   "id": "7cbc7ee0",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "dc0d07cb",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": 20,
+   "id": "a783f2b2",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": 18,
+   "id": "82c17e4b",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "b9bf03e8",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": 13,
+   "id": "c55c2608",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": 14,
+   "id": "954f3de0",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": 15,
+   "id": "addc4b16",
    "metadata": {},
    "outputs": [
     {
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "5fdc4a48",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "b606622f",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "e123b1d7",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "fae5c564",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "54799082",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "06fbb196",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "566e5a39",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "fd175745",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "d04bf31b",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "4211ee77",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "189ed063",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "ba901e35",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "2276d112",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "aa384bc1",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "986c64ca",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "7c605acc",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "690bc8ad",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "018d4db4",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "bf21ade3",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "bc3d51f6",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "10da50d4",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "cc80c5e1",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "d2ef3fd1",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "9721b6d1",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "a5fbfb1e",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "b60cca21",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "acaed425",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "c7c363e7",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "a87ab1b7",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "49d2d652",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "41c258c7",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "2112d785",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "9a21694d",
    "metadata": {},
    "outputs": [],
    "source": []
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "55d3ccb7",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "072d4638",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "c984e701",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "c9081831",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "5aa598fa",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "c16df369",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "dde802e6",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "4b36a671",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "9916d649",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "024c43da",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "58ad756c",
    "metadata": {
     "scrolled": true
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "5ac35907",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "1f9a585d",
    "metadata": {
     "scrolled": false
    },
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "7c0d9839",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "956c1225",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "d6efa7a5",
    "metadata": {},
    "outputs": [],
    "source": [
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "80181787",
    "metadata": {},
    "outputs": [],
    "source": []
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "0e1394e6",
    "metadata": {},
    "outputs": [],
    "source": []
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.10.4"
   }
  },
  "nbformat": 4,

requirements.txt

 pymongo
 dnspython
 ttp
+humanize