indoor_localisation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-J7zCx6WbY4v"
      },
      "outputs": [],
      "source": [
        "import tensorflow\n",
        "import numpy\n",
        "import prettytable\n",
        "import networkx\n",
        "import numba\n",
        "import sklearn\n",
        "import matplotlib"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "1Hyq6XNgj2CF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import shutil\n",
        "\n",
        "target_folder = [\"sites\", \"output_data\"]\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    #root = os.getcwd()\n",
        "    root = \"/content/drive/MyDrive/\"\n",
        "    data_folder = os.path.join(os.path.dirname(root), \"Floors/data\")\n",
        "    for dir_name in target_folder:\n",
        "        dir_path = os.path.join(data_folder, dir_name)\n",
        "        if os.path.isdir(dir_path):\n",
        "            shutil.rmtree(dir_path)\n",
        "        os.mkdir(dir_path)"
      ],
      "metadata": {
        "id": "MGQN7UEgcigC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import os\n",
        "import json\n",
        "#import sys\n",
        "#sys.path.insert(0,\"/content/drive/MyDrive/Floors\")\n",
        "from const import *\n",
        "\n",
        "settings_id = \"preprocess\"\n",
        "settings_node_id = \"preprocess node\"\n",
        "settings_file = \"/content/drive/MyDrive/Floors/settings.json\"\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    input_dir_name = \"\"\n",
        "    output_dir_name = \"\"\n",
        "    building_id = \"\"\n",
        "    with open(settings_file, 'r') as f_in:\n",
        "        all_settings = json.load(f_in)\n",
        "        settings = all_settings[settings_id]\n",
        "        input_dir_name = settings[\"raw input dir\"]\n",
        "        output_dir_name = settings[\"node output dir\"]\n",
        "\n",
        "    #root = os.getcwd()\n",
        "    root = \"/content/drive/MyDrive/Floors/\"\n",
        "    for building_id in building_ids:\n",
        "        print (f\"building_id: {building_id}\")\n",
        "        train_path = os.path.abspath(os.path.join(root, input_dir_name, building_id))\n",
        "        for floor_id in os.listdir(train_path): # loop through each floor\n",
        "            floor_path = os.path.abspath(os.path.join(train_path, floor_id))\n",
        "            breakpoint_info_list = []\n",
        "            for filename in os.listdir(floor_path): # loop through all files in each floor\n",
        "                file_path = os.path.abspath(os.path.join(floor_path, filename))\n",
        "                with open(file_path, 'rb') as f_in:\n",
        "                    for line in f_in:\n",
        "                        line_decoded = line.decode(\"utf-8\")\n",
        "                        if ('TYPE_WAYPOINT' in line_decoded):\n",
        "                            waypoint_info = line_decoded.split('\\t')\n",
        "                            breakpoint_info_list.append([waypoint_info[2], waypoint_info[3].strip(), waypoint_info[0]])\n",
        "            breakpoint_list = sorted(breakpoint_info_list, key=lambda x:int(x[2]))\n",
        "\n",
        "            output_folder = os.path.abspath(os.path.join(root, output_dir_name, building_id))\n",
        "            if not (os.path.exists(output_folder)):\n",
        "                os.mkdir(output_folder)\n",
        "            output_file = os.path.abspath(os.path.join(output_folder, \"{}_WiFi.txt\".format(floor_id)))\n",
        "\n",
        "            with open(file_path, 'rb') as f_in, open(output_file, \"a\") as f_out:\n",
        "                f_out.write(\"Start: \\n\") # first line\n",
        "                for i in range(len(breakpoint_list)): # second line\n",
        "                    f_out.write(\"{},{} \".format(breakpoint_list[i][0], breakpoint_list[i][1]))\n",
        "                f_out.write(\"\\n\")\n",
        "                for i in range(len(breakpoint_list)): # third line\n",
        "                    f_out.write(\"{} \".format(breakpoint_list[i][2]))\n",
        "                f_out.write(\"\\n\")\n",
        "\n",
        "            current_time = 0\n",
        "            first_time = True\n",
        "            for filename in os.listdir(floor_path):\n",
        "                with open(file_path, 'rb') as f_in, open(output_file, \"a\") as f_out:\n",
        "                    for line in f_in:\n",
        "                        line_decoded = line.decode(\"utf-8\")\n",
        "                        if ('TYPE_WIFI' in line_decoded):\n",
        "                            wifi_info = line_decoded.split('\\t')\n",
        "                            if int(wifi_info[0])!=current_time:\n",
        "                                if first_time:\n",
        "                                    first_time = False\n",
        "                                else:\n",
        "                                    f_out.write(\"\\n\")\n",
        "                                f_out.write(\"{} \".format(wifi_info[0]))\n",
        "                                current_time = int(wifi_info[0])\n",
        "                            f_out.write(\"{},{} \".format(wifi_info[2], wifi_info[4]))"
      ],
      "metadata": {
        "id": "HXA467upj6PM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#copied from FYT/wifine\n",
        "#import os\n",
        "#import json\n",
        "import pickle\n",
        "#import shutil\n",
        "\n",
        "#from utils import *\n",
        "#from const import *\n",
        "\n",
        "#utils.py module\n",
        "\n",
        "def load_file_comparison(filename, observation_ids, observation_sign, file_type, prune):\n",
        "    max_id = 0\n",
        "    if observation_ids:\n",
        "        max_id = max(max_id, int(observation_ids[-1][0][1:]))\n",
        "    mac_all = []\n",
        "    with open(filename, \"r\") as f_in:\n",
        "        while True:\n",
        "            line = f_in.readline().rstrip(\" \\n\")\n",
        "            if not line:\n",
        "                break\n",
        "            if file_type == \"path\":\n",
        "                if line.startswith(\"Start:\"):\n",
        "                    device_id = os.path.basename(filename)[:2]\n",
        "                    try:\n",
        "                        breakpoints = [[float(coor) for coor in item.split(\n",
        "                            \",\")] for item in f_in.readline().rstrip(\" \\n\").split(\" \")]\n",
        "                        timestamps = [\n",
        "                            int(ts) for ts in f_in.readline().rstrip(\" \\n\").split(\" \")]\n",
        "                    except ValueError:\n",
        "                        break\n",
        "                    continue\n",
        "                timestamp, rssi_pairs = line.split(\" \", 1)\n",
        "                ground_truth = interpolate_point(\n",
        "                    int(timestamp), timestamps, breakpoints)\n",
        "            elif file_type == \"db\":\n",
        "                device_id = os.path.basename(filename)\n",
        "                coor, rssi_pairs = line.split(\" \", 1)\n",
        "                ground_truth = [float(item) for item in coor.split(\",\")]\n",
        "            elif file_type == \"new\":\n",
        "                wifi_json = json.loads(line)\n",
        "                timestamp = wifi_json['sysTimeMs']\n",
        "                rssi_pairs = ''\n",
        "                for item in wifi_json['data']:\n",
        "                    rssi_pairs += str(item['bssid'].replace(':','')) + ',' + str(item['rssi']) + ' '\n",
        "                rssi_pairs = rssi_pairs.strip(' ')\n",
        "                ground_truth = [None,None]\n",
        "                device_id = None\n",
        "\n",
        "            rssi_dict = {}\n",
        "            for rssi_pair in rssi_pairs.split(\" \"):\n",
        "                mac = rssi_pair.split(\",\")[0]\n",
        "                rssi = rssi_pair.split(\",\")[1]\n",
        "                # remove virtual mac\n",
        "                if prune and is_virtual_mac(mac):\n",
        "                    continue\n",
        "                try:\n",
        "                    rssi_dict[mac] = float(rssi)\n",
        "                except ValueError:\n",
        "                    continue\n",
        "                if mac not in mac_all:\n",
        "                    mac_all.append(mac)\n",
        "            if rssi_dict:\n",
        "                observation_ids.append(\n",
        "                    [\"{}{}\".format(observation_sign, max_id+1), device_id, ground_truth, rssi_dict])\n",
        "                max_id += 1\n",
        "\n",
        "    print(\"{} loaded\".format(filename))\n",
        "    return observation_ids, mac_all\n",
        "\n",
        "\n",
        "def series_to_file(obj, filename):\n",
        "    with open(filename, 'wb') as f_out:\n",
        "        pickle.dump(obj, f_out, -1)\n",
        "        print(\"Data written into {}\".format(filename))\n",
        "\n",
        "\n",
        "def file_to_series(filename):\n",
        "    with open(filename, 'rb') as f_in:\n",
        "        series = pickle.load(f_in)\n",
        "        print(\"File {} loaded.\".format(filename))\n",
        "        return series\n",
        "\n",
        "\n",
        "def rssi2weight(offset, rssi):\n",
        "    return offset + rssi\n",
        "\n",
        "\n",
        "def is_virtual_mac(mac_addr):\n",
        "    mac_addr = mac_addr.replace(\":\", \"\").upper()\n",
        "    first_hex = int(mac_addr[0:2], 16)\n",
        "    return first_hex & 0x02 != 0\n",
        "\n",
        "\n",
        "def interpolate_point(timestamp, timestamps, breakpoints):\n",
        "    if timestamp <= timestamps[0]:\n",
        "        print(\"timestamp too small: {} <= {}\".format(timestamp, timestamps[0]))\n",
        "        return breakpoints[0]\n",
        "    if timestamp >= timestamps[-1]:\n",
        "        print(\"timestamp too large: {} >= {}\".format(\n",
        "            timestamp, timestamps[-1]))\n",
        "        return breakpoints[-1]\n",
        "\n",
        "    for idx in range(len(timestamps)-1):\n",
        "        if timestamps[idx] <= timestamp <= timestamps[idx+1]:\n",
        "            return [breakpoints[idx][coor_id] + (timestamp - timestamps[idx]) /\n",
        "                    (timestamps[idx+1] - timestamps[idx]) *\n",
        "                    (breakpoints[idx+1][coor_id] - breakpoints[idx][coor_id])\n",
        "                    for coor_id in [0, 1]]"
      ],
      "metadata": {
        "id": "hpzwsm9RoNMs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# First step: load file and get observation points\n",
        "def load_file(filename, observation_ids, observation_sign, file_type, prune, th):\n",
        "    print (filename)\n",
        "    max_id = 0\n",
        "    if observation_ids:\n",
        "        max_id = max(max_id, int(observation_ids[-1][0][1:]))\n",
        "\n",
        "    with open(filename, \"r\") as f_in:\n",
        "        while True:\n",
        "            line = f_in.readline().rstrip(\" \\n\")\n",
        "            if not line:\n",
        "                break\n",
        "            if file_type == \"path\":\n",
        "                if line.startswith(\"Start:\"):\n",
        "                    device_id = os.path.basename(filename)[:2]\n",
        "                    try:\n",
        "                        breakpoints = [[float(coor) for coor in item.split(\n",
        "                            \",\")] for item in f_in.readline().rstrip(\" \\n\").split(\" \")]\n",
        "                        timestamps = [\n",
        "                            int(ts) for ts in f_in.readline().rstrip(\" \\n\").split(\" \")]\n",
        "                    except ValueError:\n",
        "                        break\n",
        "                    continue\n",
        "                timestamp, rssi_pairs = line.split(\" \", 1)\n",
        "                ground_truth = interpolate_point(\n",
        "                    int(timestamp), timestamps, breakpoints)\n",
        "            elif file_type == \"db\":\n",
        "                device_id = os.path.basename(filename)\n",
        "                coor, rssi_pairs = line.split(\" \", 1)\n",
        "                ground_truth = [float(item) for item in coor.split(\",\")]\n",
        "            elif file_type == \"new\":\n",
        "                wifi_json = json.loads(line)\n",
        "                timestamp = wifi_json['sysTimeMs']\n",
        "                rssi_pairs = ''\n",
        "                for item in wifi_json['data']:\n",
        "                    rssi_pairs += str(item['bssid'].replace(':','')) + ',' + str(item['rssi']) + ' '\n",
        "                rssi_pairs = rssi_pairs.strip(' ')\n",
        "                ground_truth = [None,None]\n",
        "                device_id = None\n",
        "            rssi_dict = {}\n",
        "            for rssi_pair in rssi_pairs.split(\" \"):\n",
        "                try:\n",
        "                    mac = rssi_pair.split(\",\")[0]\n",
        "                    rssi = rssi_pair.split(\",\")[1]\n",
        "                    # remove virtual mac\n",
        "                    if prune and is_virtual_mac(mac):\n",
        "                        continue\n",
        "                    if float(rssi) >= th:\n",
        "                        rssi_dict[mac] = float(rssi)\n",
        "                except ValueError:\n",
        "                    continue\n",
        "                except IndexError:\n",
        "                    continue\n",
        "            if rssi_dict:\n",
        "                observation_ids.append(\n",
        "                    [\"{}{}\".format(observation_sign, max_id+1), device_id, ground_truth, rssi_dict])\n",
        "                max_id += 1\n",
        "\n",
        "    print(\"{} loaded\".format(filename))\n",
        "    return observation_ids"
      ],
      "metadata": {
        "id": "1CYpVx7GqE68"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Second step: generate ap nodes from given observation points\n",
        "def generate_ap_ids(ap_file, ap_sign, observation_ids):\n",
        "    max_id = 0\n",
        "    if os.path.isfile(ap_file):\n",
        "        ap_ids = file_to_series(ap_file)\n",
        "        max_id = max(max_id, int(ap_ids[-1][0][1:]))\n",
        "    else:\n",
        "        ap_ids = []\n",
        "\n",
        "    for observation in observation_ids:\n",
        "        for mac in observation[3].keys():\n",
        "            ap_flag = False\n",
        "            for ap in ap_ids:\n",
        "                if ap[1] == mac:\n",
        "                    ap_flag = True\n",
        "                    ap[3].append([observation[3][mac]]+observation[2])\n",
        "                    break\n",
        "            if not ap_flag:\n",
        "                ap_ids.append([\"{}{}\".format(ap_sign, max_id+1), mac,\n",
        "                               None, [[observation[3][mac]]+observation[2]]])\n",
        "                max_id += 1\n",
        "    if ap_ids:\n",
        "        series_to_file(ap_ids, ap_file)\n",
        "    else:\n",
        "        print(\"No AP ids!\")\n",
        "    return ap_ids"
      ],
      "metadata": {
        "id": "Akcajel3qL6L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Third step: use observation nodes and ap nodes to generate edgelist for further use\n",
        "def generate_graph_file(observation_file, ap_file, offset, output_file):\n",
        "    observation_ids = file_to_series(observation_file)\n",
        "    ap_ids = file_to_series(ap_file)\n",
        "    ap_dict = {}\n",
        "    for ap_item in ap_ids:\n",
        "        ap_dict[ap_item[1]] = ap_item[0]\n",
        "\n",
        "    with open(output_file, \"w\") as f_out:\n",
        "        for observation in observation_ids:\n",
        "            ob_id = observation[0]\n",
        "            for mac in observation[3].keys():\n",
        "                f_out.write(\"{} {} {}\\n\".format(ob_id, ap_dict[mac], rssi2weight(\n",
        "                    offset, observation[3][mac])))\n",
        "    print(\"edgelist generated.\")\n",
        "\n",
        "settings_id = \"preprocess\"\n",
        "settings_file = \"/content/drive/MyDrive/Floors/settings.json\"\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # load settings\n",
        "    offset, ap_sign, observation_sign, th = 0, \"\", \"\", 0\n",
        "    with open(settings_file, 'r') as f_in:\n",
        "        all_settings = json.load(f_in)\n",
        "        settings = all_settings[settings_id]\n",
        "        offset = settings['offset']\n",
        "        ap_sign = settings['ap sign']\n",
        "        observation_sign = settings['observation sign']\n",
        "        th = settings['threshold']\n",
        "        input_dir_name = settings[\"node output dir\"]\n",
        "        output_dir_name = settings[\"graph output parent dir\"]\n",
        "        target_dir_name = settings[\"graph output dir\"]\n",
        "\n",
        "    print(\"processing...\")\n",
        "    info_dict = {}\n",
        "    #root_dir = os.getcwd()\n",
        "    root_dir = \"/content/drive/MyDrive/Floors/\"\n",
        "    input_dir = os.path.join(root_dir, input_dir_name)\n",
        "    output_dir = os.path.join(root_dir, output_dir_name)\n",
        "    # reset the output directory\n",
        "    if not os.path.isdir(output_dir):\n",
        "        os.mkdir(output_dir)\n",
        "    # else:\n",
        "    #     shutil.rmtree(output_dir)\n",
        "    #     os.mkdir(output_dir)\n",
        "\n",
        "    for raw_data_dir in os.listdir(input_dir):\n",
        "        item = os.path.join(input_dir, raw_data_dir)\n",
        "        if raw_data_dir not in building_ids:\n",
        "            continue\n",
        "        if os.path.isdir(item):\n",
        "            output_dir_dataset = os.path.join(output_dir, raw_data_dir)\n",
        "            target_dir = os.path.join(output_dir_dataset, target_dir_name)\n",
        "            if not os.path.isdir(output_dir_dataset):\n",
        "                os.mkdir(output_dir_dataset)\n",
        "                os.mkdir(target_dir)\n",
        "                os.mkdir(os.path.join(output_dir_dataset, \"embedding\"))\n",
        "                os.mkdir(os.path.join(output_dir_dataset, \"anchor list\"))\n",
        "                os.mkdir(os.path.join(output_dir_dataset, \"spring\"))\n",
        "                os.mkdir(os.path.join(output_dir_dataset, \"anchor map\"))\n",
        "                os.mkdir(os.path.join(output_dir_dataset, \"incremental data\"))\n",
        "\n",
        "            observation_file = os.path.join(target_dir, \"_observations_{}.pkl\".format(raw_data_dir))\n",
        "            ap_file = os.path.join(target_dir, \"_APs_{}.pkl\".format(raw_data_dir))\n",
        "\n",
        "            pruned_observation_file = os.path.join(target_dir, \"_observations_prune_{}.pkl\".format(raw_data_dir))\n",
        "            pruned_ap_file = os.path.join(target_dir, \"_APs_prune_{}.pkl\".format(raw_data_dir))\n",
        "\n",
        "            graph_file = os.path.join(target_dir, \"{}.edgelist\".format(raw_data_dir))\n",
        "            pruned_graph_file = os.path.join(target_dir, \"prune_{}.edgelist\".format(raw_data_dir))\n",
        "\n",
        "            observation_ids = []\n",
        "            pruned_observation_ids = []\n",
        "            for filename in os.listdir(item):\n",
        "                if filename.endswith(\"WiFi.txt\"):\n",
        "                    file = os.path.join(item, filename)\n",
        "                    observation_ids = load_file(file, observation_ids, observation_sign, file_type=\"path\", prune=False, th=th)\n",
        "                    pruned_observation_ids = load_file(file, pruned_observation_ids, observation_sign, file_type=\"path\", prune=True, th=th)\n",
        "                elif filename.startswith(\"fingerprint\"):\n",
        "                    file = os.path.join(item, filename)\n",
        "                    observation_ids = load_file(file, observation_ids, observation_sign, file_type=\"db\", prune=False, th=th)\n",
        "                    pruned_observation_ids = load_file(file, pruned_observation_ids, observation_sign, file_type=\"db\", prune=True, th=th)\n",
        "                elif filename.startswith(\"wifi-\"):\n",
        "                    file = os.path.join(item, filename)\n",
        "                    observation_ids = load_file(file, observation_ids, observation_sign, file_type=\"new\", prune=False, th=th)\n",
        "                    pruned_observation_ids = load_file(file, pruned_observation_ids, observation_sign, file_type=\"new\", prune=True, th=th)\n",
        "\n",
        "            series_to_file(observation_ids, observation_file)\n",
        "            series_to_file(pruned_observation_ids, pruned_observation_file)\n",
        "\n",
        "            ap_ids = generate_ap_ids(ap_file, ap_sign, observation_ids)\n",
        "\n",
        "            pruned_ap_ids = generate_ap_ids(pruned_ap_file, ap_sign, pruned_observation_ids)\n",
        "\n",
        "            generate_graph_file(observation_file, ap_file, offset, graph_file)\n",
        "            generate_graph_file(pruned_observation_file, pruned_ap_file, offset, pruned_graph_file)\n",
        "\n",
        "            info_dict[raw_data_dir] = {}\n",
        "            info_dict[raw_data_dir][\"observation\"] = len(observation_ids)\n",
        "            info_dict[raw_data_dir][\"ap\"] = len(ap_ids)\n",
        "            info_dict[raw_data_dir][\"prune observation\"] = len(pruned_observation_ids)\n",
        "            info_dict[raw_data_dir][\"prune ap\"] = len(pruned_ap_ids)\n",
        "\n",
        "    print(\"generation completed.\")\n",
        "    print(info_dict)"
      ],
      "metadata": {
        "id": "oLHyLV4lqMEJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import os\n",
        "#import pickle5 as pickle #pickle already imported\n",
        "#import json\n",
        "#rom const import *\n",
        "\n",
        "#root_dir = os.getcwd()\n",
        "root_dir = \"/content/drive/MyDrive/Floors/\"\n",
        "\n",
        "for building_id in building_ids:\n",
        "    raw_path = os.path.abspath(os.path.join(root_dir, \"./data/output_data/{}/raw_data\").format(building_id))\n",
        "    obs_path = os.path.join(raw_path, \"_observations_prune_{}.pkl\".format(building_id))\n",
        "    out_file_floors = os.path.abspath(os.path.join(root_dir, \"./data/output_data/{}/raw_data/floors.txt\".format(building_id)))\n",
        "\n",
        "    with open(obs_path, 'rb') as f_in1, open(out_file_floors, \"w\") as f_out:\n",
        "        series_obs = pickle.load(f_in1)\n",
        "        series_obs = {series_obs[i][0]:series_obs[i][1]for i in range(len(series_obs))}\n",
        "        f_out.write(str(series_obs))"
      ],
      "metadata": {
        "id": "Fx3sMHbTrFpM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python --version\n",
        "print(tf. __version__)"
      ],
      "metadata": {
        "id": "QOcZ6037wej7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "import tensorflow as tf\n",
        "from const import *\n",
        "from graph import Graph\n",
        "from eline import E_LINE\n",
        "from paths import *\n",
        "#from utils import *\n",
        "os.environ['CUDA_VISIBLE_DEVICES']='0'\n",
        "\n",
        "def clean_main(site_id, order, rep_size, negative_ratio):\n",
        "    # step 1: preprocess, generate network edgelist for further processing\n",
        "    site_folder = os.path.abspath(os.path.join(ROOT_FOLDER, \"drive/MyDrive/Floors/\", output_dir_graph, site_id))\n",
        "    site_embedding_folder = os.path.join(site_folder, \"embedding\")\n",
        "    observation_ids = file_to_series(os.path.join(site_folder, \"raw_data\", \"_observations_{}.pkl\".format(site_id)))\n",
        "    ap_ids = file_to_series(os.path.join(site_folder, \"raw_data\", \"_APs_{}.pkl\".format(site_id)))\n",
        "    pruned_observation_ids = file_to_series(os.path.join(site_folder, \"raw_data\", \"_observations_prune_{}.pkl\".format(site_id)))\n",
        "    prunde_ap_ids = file_to_series(os.path.join(site_folder, \"raw_data\", \"_APs_prune_{}.pkl\".format(site_id)))\n",
        "\n",
        "    # edgelist_file = gen_partial_edgelist_file(site_id, selected_ids, percent)\n",
        "    edgelist_file = os.path.join(site_folder, \"raw_data\", \"{}.edgelist\".format(site_id))\n",
        "    if not os.path.isfile(edgelist_file):\n",
        "        generate_graph_file(edgelist_file, observation_ids, ap_ids)\n",
        "\n",
        "    # step 2: E-LINE\n",
        "    embedding_file = os.path.join(site_embedding_folder, \"dim_{}.txt\".format(rep_size))\n",
        "    if not os.path.isfile(embedding_file):\n",
        "        g = Graph()\n",
        "        g.read_edgelist(filename=edgelist_file, weighted=True)\n",
        "        model = E_LINE(g, epoch=100, rep_size=rep_size, order=order, negative_ratio=negative_ratio)\n",
        "        model.save_embeddings(embedding_file)\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    order = 2\n",
        "    rep_size = 8\n",
        "    negative_ratio = 4\n",
        "\n",
        "    with tf.device('/GPU:0'):\n",
        "        for idx0 in range(len(building_ids)):\n",
        "            site_folder = os.path.abspath(os.path.join(ROOT_FOLDER, output_dir_graph, building_ids[idx0]))\n",
        "            clean_main(building_ids[idx0], order, rep_size, negative_ratio)"
      ],
      "metadata": {
        "id": "LKCnkr0VrF-0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from hierarchical_centroid import *\n",
        "from multiprocess import Pool\n",
        "from const import *\n",
        "\n",
        "dimension_of_vector = default_config['emb_dim']\n",
        "num_reference_points_each_floor = default_config['ref_num_per_floor']\n",
        "\n",
        "\n",
        "def main_hierarchical_average(reference_matrix, out_file):\n",
        "    global ground_truth_value_list\n",
        "    cluster_list = clustering_method_hierarchical_average_new(dataset, reference_matrix)\n",
        "    pred_list = generate_pred_list_hierarchical_average(cluster_list, dataset, reference_matrix, ground_truth_value_list)\n",
        "    print(\"reference matrix size is\", reference_matrix.size)\n",
        "    print(\"reference matrix is:\\n\", reference_matrix)\n",
        "    print(\"pred list is:\\n\", pred_list)\n",
        "    print(\"len of pred list is\", len(pred_list))\n",
        "    accuracy, report = assess_hierarchical_average(pred_list, ground_truth_list, ground_truth_value_list)\n",
        "    print (\"Out file: {}\".format(out_file))\n",
        "    with open(out_file, 'a') as f_out:\n",
        "        f_out.write('\\n'+str(report))\n",
        "    return accuracy, pred_list\n",
        "\n",
        "\n",
        "# Hierarchical test starts here ===================================================================\n",
        "repeat_times = 10\n",
        "acc_list = []\n",
        "total_pred_list = []\n",
        "total_truth_list = []\n",
        "\n",
        "\n",
        "def util(i):\n",
        "    global embedding_filename, ground_truth_filename, dataset, reference_matrix, ground_truth_value_list, ground_truth_list\n",
        "    file_path = ''\n",
        "    for building_id in building_ids:\n",
        "        embedding_filename = out_file_embedding.format(building_id, dimension_of_vector)\n",
        "        if not os.path.exists(embedding_filename):\n",
        "            continue\n",
        "        ground_truth_filename = out_file_floors.format(building_id)\n",
        "        out_file_undirected = out_file_report.format(building_id, dimension_of_vector)\n",
        "        dataset, desired_num_of_clusters, ground_truth_list, ground_truth_value_list, num_of_samples_in_each_floor = \\\n",
        "            get_dataset_and_truth_list_two(file_path, dimension_of_vector, embedding_filename, ground_truth_filename)\n",
        "        print(len(dataset))\n",
        "        print(len(ground_truth_list))\n",
        "\n",
        "        dataset = np.array(dataset)\n",
        "        reference_matrix = generate_reference_matrix(ground_truth_list, ground_truth_value_list,\n",
        "                                                     num_reference_points_each_floor)\n",
        "        main_hierarchical_average(reference_matrix, out_file_undirected)\n",
        "\n",
        "\n",
        "t_start = time.time()\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    p = Pool(repeat_times)\n",
        "    list_of_pred_list = p.map(util, range(repeat_times))\n",
        "    p.close()\n",
        "    p.join()\n",
        "\n",
        "t_end = time.time()\n",
        "print(\"Total time is\", (t_end - t_start)/60, \"min\")"
      ],
      "metadata": {
        "id": "yu8U9bZ2rGaT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}