Preprocessing_JSON.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_qhF84nV0mt",
        "outputId": "8dcf1690-e63e-4ba8-cf99-30394ea00d37"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "with open('/content/topiocqa_dev.json') as file:\n",
        "  data = json.load(file)\n",
        "#print(data)"
      ],
      "metadata": {
        "id": "uhUbXR0vXOla"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "with open('topiocqa_train.json') as file:\n",
        "  data = json.load(file)\n",
        "#print(data)"
      ],
      "metadata": {
        "id": "rUde4YVSV_sW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(data))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nCIJVIODZ0Qq",
        "outputId": "d720ec2e-5a75-4349-8e8c-5f821686f680"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "45450\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Current Topic, Context List (Question Answer)\n",
        "\n",
        "# data_prompt_completion = []\n",
        "\n",
        "# for conversation in data:\n",
        "#   context = conversation[\"Context\"]\n",
        "#   prompt_completetion_instance = {}\n",
        "#   prompt = \"\"\n",
        "#   current_topic = conversation[\"Topic\"]\n",
        "\n",
        "#   prompt += \"Topic: \" + current_topic + \"\\n\\n###\\n\\n\"\n",
        "\n",
        "#   for i, qa in enumerate(context):\n",
        "#     if i%2 == 0:\n",
        "#       prompt += \"Question: \" + qa + \"\\n\"\n",
        "#     else:\n",
        "#       prompt += \"Answer: \" + qa + \"\\n\"\n",
        "\n",
        "#   question = conversation[\"Question\"]\n",
        "#   prompt += \"Question: \" + question + \"\\nAnswer:\"\n",
        "\n",
        "#   prompt_completetion_instance[\"prompt\"] = prompt\n",
        "\n",
        "#   answer = conversation[\"Answer\"]\n",
        "#   completition = \" \" + answer + \"\\n\"\n",
        "#   prompt_completetion_instance[\"completion\"] = completition\n",
        "\n",
        "#   data_prompt_completion.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "GQb8f9PTFOYz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Current Topic, Context List (Question Answer) Alternate\n",
        "\n",
        "data_prompt_completion = []\n",
        "answer_length = 0\n",
        "max_answer = ''\n",
        "total_length = 0\n",
        "for conversation in data:\n",
        "  context = conversation[\"Context\"]\n",
        "  prompt_completetion_instance = {}\n",
        "  prompt = \"\"\n",
        "  current_topic = conversation[\"Topic\"]\n",
        "\n",
        "  prompt += \"Topic: \" + current_topic + \"\\n\\n\"\n",
        "\n",
        "  for i, qa in enumerate(context):\n",
        "    if i%2 == 0:\n",
        "      prompt += \"Question: \" + qa + \"\\n\"\n",
        "    else:\n",
        "      prompt += \"Answer: \" + qa + \"\\n\"\n",
        "\n",
        "  question = conversation[\"Question\"]\n",
        "  prompt += \"Question: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "  prompt_completetion_instance[\"prompt\"] = prompt\n",
        "\n",
        "  answer = conversation[\"Answer\"]\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance[\"completion\"] = completition\n",
        "  if answer.count(' ') > answer_length:\n",
        "    answer_length=answer.count(' ')\n",
        "    max_answer = answer\n",
        "  total_length += answer.count(' ')\n",
        "\n",
        "  data_prompt_completion.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "qUCNwMf7WWQN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_length"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jOJtsB2uWWH5",
        "outputId": "484a0073-d1d7-4493-9bc5-5d92bc8058de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "144"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_answer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        },
        "id": "O2YWWLPj7ESj",
        "outputId": "b9eea751-52ea-440e-ceee-08f31f5de5aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'As a storm approaches a southern Louisiana bayou community called the \"Bathtub\" (a community cut off from the rest of the world by a levee), six-year-old Hushpuppy and her ailing, hot-tempered father Wink are optimistic about their life and their future. The children in school are being taught by Miss Bathsheba about nature and the release of prehistoric aurochs from the melting ice caps. At home, Hushpuppy fends for herself while her father is missing. When he returns, he is wearing a hospital gown and bracelet. They argue, and when Hushpuppy returns to her house, she deliberately sets it on fire. A chase ensues between the two, and she ends up getting slapped by Wink. When she retaliates by punching him in the chest, Wink collapses. Hushpuppy, realizing the damage she has caused, runs for help only to find her father missing when she returns.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_length/(len(data))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mzkXXXxb7Zbg",
        "outputId": "ba04e6a3-3a0e-4d5c-9802-15b2b5531b07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9.89078107810781"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        " #data_prompt_completion\n",
        "!pip install jsonlines\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_context_topic_data.jsonl', 'w') as writer:\n",
        "    writer.write_all(data_prompt_completion)\n",
        "\n",
        "files.download('gpt3_dev_context_topic_data.jsonl')"
      ],
      "metadata": {
        "id": "bCRc5sD6PStO",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "228576fe-ca8a-4b4d-d72b-8e2611680338"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jsonlines\n",
            "  Downloading jsonlines-3.1.0-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3021, in _dep_map\n",
            "    return self.__dep_map\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2815, in __getattr__\n",
            "    raise AttributeError(attr)\n",
            "AttributeError: _DistInfoDistribution__dep_map\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/base_command.py\", line 160, in exc_logging_wrapper\n",
            "    status = run_func(*args)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/req_command.py\", line 241, in wrapper\n",
            "    return func(self, options, args)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/commands/install.py\", line 499, in run\n",
            "    conflicts = self._determine_conflicts(to_install)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/commands/install.py\", line 631, in _determine_conflicts\n",
            "    return check_install_conflicts(to_install)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/operations/check.py\", line 101, in check_install_conflicts\n",
            "    package_set, _ = create_package_set_from_installed()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/operations/check.py\", line 42, in create_package_set_from_installed\n",
            "    dependencies = list(dist.iter_dependencies())\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/metadata/pkg_resources.py\", line 216, in iter_dependencies\n",
            "    return self._dist.requires(extras)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 2736, in requires\n",
            "    dm = self._dep_map\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3023, in _dep_map\n",
            "    self.__dep_map = self._compute_dependencies()\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3033, in _compute_dependencies\n",
            "    reqs.extend(parse_requirements(req))\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3094, in parse_requirements\n",
            "    yield Requirement(line)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pkg_resources/__init__.py\", line 3101, in __init__\n",
            "    super(Requirement, self).__init__(requirement_string)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/packaging/requirements.py\", line 102, in __init__\n",
            "    req = REQUIREMENT.parseString(requirement_string)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 1131, in parse_string\n",
            "    loc, tokens = self._parse(instring, 0)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n",
            "    loc, exprtokens = e._parse(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4114, in parseImpl\n",
            "    return e._parse(\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n",
            "    loc, exprtokens = e._parse(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4959, in parseImpl\n",
            "    loc, tokens = self_expr._parse(instring, loc, doActions, callPreParse=False)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n",
            "    loc, exprtokens = e._parse(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n",
            "    loc, exprtokens = e._parse(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 5226, in parseImpl\n",
            "    return super().parseImpl(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4375, in parseImpl\n",
            "    return self.expr._parse(instring, loc, doActions, callPreParse=False)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3864, in parseImpl\n",
            "    loc, resultlist = self.exprs[0]._parse(\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4114, in parseImpl\n",
            "    return e._parse(\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4375, in parseImpl\n",
            "    return self.expr._parse(instring, loc, doActions, callPreParse=False)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 3886, in parseImpl\n",
            "    loc, exprtokens = e._parse(instring, loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 817, in _parseNoCache\n",
            "    loc, tokens = self.parseImpl(instring, pre_loc, doActions)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 4114, in parseImpl\n",
            "    return e._parse(\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 856, in _parseNoCache\n",
            "    tokens = fn(instring, tokens_start, ret_tokens)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/core.py\", line 291, in wrapper\n",
            "    ret = func(*args[limit:])\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/packaging/markers.py\", line 122, in <lambda>\n",
            "    MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0]))\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/pyparsing/results.py\", line 193, in __getitem__\n",
            "    return self._toklist[i]\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/pip3\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/main.py\", line 70, in main\n",
            "    return command.main(cmd_args)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/base_command.py\", line 101, in main\n",
            "    return self._main(args)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/base_command.py\", line 214, in _main\n",
            "    return run(options, args)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/cli/base_command.py\", line 197, in exc_logging_wrapper\n",
            "    logger.critical(\"Operation cancelled by user\")\n",
            "  File \"/usr/lib/python3.9/logging/__init__.py\", line 1493, in critical\n",
            "    self._log(CRITICAL, msg, args, **kwargs)\n",
            "  File \"/usr/lib/python3.9/logging/__init__.py\", line 1589, in _log\n",
            "    self.handle(record)\n",
            "  File \"/usr/lib/python3.9/logging/__init__.py\", line 1599, in handle\n",
            "    self.callHandlers(record)\n",
            "  File \"/usr/lib/python3.9/logging/__init__.py\", line 1661, in callHandlers\n",
            "    hdlr.handle(record)\n",
            "  File \"/usr/lib/python3.9/logging/__init__.py\", line 952, in handle\n",
            "    self.emit(record)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_internal/utils/logging.py\", line 169, in emit\n",
            "    renderable = self.render_message(record, message)\n",
            "  File \"/usr/local/lib/python3.9/dist-packages/pip/_vendor/rich/logging.py\", line 188, in render_message\n",
            "    if highlighter:\n",
            "KeyboardInterrupt\n",
            "^C\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-3cade65a9e92>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#data_prompt_completion\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'pip install jsonlines'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mjsonlines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'jsonlines'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jsonlines"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qCSQjEjCIABN",
        "outputId": "54c41bb3-5ef1-448f-c43f-e34261ce9efa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jsonlines\n",
            "  Using cached jsonlines-3.1.0-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n",
            "Installing collected packages: jsonlines\n",
            "Successfully installed jsonlines-3.1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_context_topic = random.sample(data_prompt_completion, 500)\n",
        "\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_context_topic_data_trial.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_context_topic)\n",
        "\n",
        "files.download('gpt3_dev_context_topic_data_trial.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "OtcEK2UKOfP6",
        "outputId": "327eee61-b921-4009-bb34-d5e823309f2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_650c5388-18dc-46ef-aa87-c4d3dcbab6c0\", \"gpt3_dev_context_topic_data_trial.jsonl\", 470921)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Previous Topic: Current Topic, Question Answer\n",
        "\n",
        "data_prompt_completion_2 = []\n",
        "\n",
        "current_topic = \"\"\n",
        "previous_topic = \"\"\n",
        "\n",
        "\n",
        "for conversation in data:\n",
        "  context = conversation[\"Context\"]\n",
        "  prompt_completetion_instance = {}\n",
        "  prompt = \"\"\n",
        "  if len(context) == 0:\n",
        "    previous_topic = \"\"\n",
        "  else:\n",
        "    previous_topic = current_topic\n",
        "  current_topic = conversation[\"Topic\"]\n",
        "\n",
        "  \n",
        "  prompt += \"Previous: \" + previous_topic + \"\\n\\n\"\n",
        "  prompt += \"Current: \" + current_topic + \"\\n\\n\"\n",
        "\n",
        "  question = conversation[\"Question\"]\n",
        "  prompt += \"Question: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "  prompt_completetion_instance[\"prompt\"] = prompt\n",
        "\n",
        "  answer = conversation[\"Answer\"]\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance[\"completion\"] = completition\n",
        "\n",
        "  data_prompt_completion_2.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "ngQffyowSBB7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_topic_topic_data.jsonl', 'w') as writer:\n",
        "    writer.write_all(data_prompt_completion_2)\n",
        "\n",
        "files.download('gpt3_dev_topic_topic_data.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "UQHb3uAEUDg5",
        "outputId": "5d306677-1408-4b4f-eb95-e15559c99bba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_7016e205-d7aa-40a9-b172-4edda414ca96\", \"gpt3_dev_topic_topic_data.jsonl\", 583266)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_topic_topic = random.sample(data_prompt_completion_2, 250)\n",
        "\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_topic_topic_data_trial.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_topic_topic)\n",
        "\n",
        "files.download('gpt3_dev_topic_topic_data_trial.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Ho05zh0iNwjQ",
        "outputId": "ce79a4d1-e756-417d-acdd-15b1919c22ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_1bb5f823-f16b-4dc2-a454-6dfbd3957993\", \"gpt3_dev_topic_topic_data_trial.jsonl\", 56207)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Previous Topic: Current Topic, Context List (Question Answer)\n",
        "\n",
        "data_prompt_completion_3 = []\n",
        "\n",
        "current_topic = \"\"\n",
        "previous_topic = \"\"\n",
        "\n",
        "\n",
        "for conversation in data:\n",
        "  context = conversation[\"Context\"]\n",
        "  prompt_completetion_instance = {}\n",
        "  prompt = \"\"\n",
        "  if len(context) == 0:\n",
        "    previous_topic = \"\"\n",
        "  else:\n",
        "    previous_topic = current_topic\n",
        "  current_topic = conversation[\"Topic\"]\n",
        "\n",
        "  \n",
        "  prompt += \"Previous: \" + previous_topic + \"\\n\\n\"\n",
        "  prompt += \"Current: \" + current_topic + \"\\n\\n\"\n",
        "\n",
        "  for i, qa in enumerate(context):\n",
        "    if i%2 == 0:\n",
        "      prompt += \"Question: \" + qa + \"\\n\"\n",
        "    else:\n",
        "      prompt += \"Answer: \" + qa + \"\\n\"\n",
        "\n",
        "  question = conversation[\"Question\"]\n",
        "  prompt += \"Question: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "  prompt_completetion_instance[\"prompt\"] = prompt\n",
        "\n",
        "  answer = conversation[\"Answer\"]\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance[\"completion\"] = completition\n",
        "\n",
        "  data_prompt_completion_3.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "WbmMzNGxV9r7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_prompt_completion_3"
      ],
      "metadata": {
        "id": "6rLLh5PxWzUA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jsonlines"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1104cb39-8aba-4803-a411-9d01aaa792cd",
        "id": "FNXDdS0kQWJI"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: jsonlines in /usr/local/lib/python3.9/dist-packages (3.1.0)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_context_topic_topic_data.jsonl', 'w') as writer:\n",
        "    writer.write_all(data_prompt_completion_3)\n",
        "\n",
        "files.download('gpt3_dev_context_topic_topic_data.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "766VbZy3QWJJ",
        "outputId": "75e58af8-64a1-4b4c-ad58-d83a9414dffb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_ec4bbd89-1b20-4e0d-bb6d-60ba848bf4a4\", \"gpt3_dev_context_topic_topic_data.jsonl\", 2382997)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install jsonlines"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AvZwP--e6v9e",
        "outputId": "90dcc829-c95a-4f39-8ec7-e2615e2e513c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jsonlines\n",
            "  Downloading jsonlines-3.1.0-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n",
            "Installing collected packages: jsonlines\n",
            "Successfully installed jsonlines-3.1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_context_topic_topic = random.sample(data_prompt_completion_3, 1000)\n",
        "\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_context_topic_topic_data.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_context_topic_topic)\n",
        "\n",
        "files.download('gpt3_dev_context_topic_topic_data.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "L0mUA0ZTIX5I",
        "outputId": "23ddb373-cfb9-4e47-8822-a7b00b10df85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_a5ecb7c0-ca9c-46ea-b1ee-7fdfa6ebb5c1\", \"gpt3_dev_context_topic_topic_data.jsonl\", 965821)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_context_topic_topic = random.sample(data_prompt_completion_3, 200)\n",
        "\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_context_topic_topic_data_trial.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_context_topic_topic)\n",
        "\n",
        "files.download('gpt3_dev_context_topic_topic_data_trial.jsonl')"
      ],
      "metadata": {
        "id": "reGfFcZE9kr_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7zulMSrxWXK6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_prompt_completion_5 = []\n",
        "previous_topic = \"\"\n",
        "previous_question = \"\"\n",
        "previous_answer = \"\"\n",
        "current_topic = \"\"\n",
        "prompt = \"\"\n",
        "for conversation in data:\n",
        "  prompt_completetion_instance_5 = {}\n",
        "\n",
        "\n",
        "  if len(conversation[\"Context\"]) == 0:\n",
        "    previous_topic = \"\"\n",
        "    previous_question = \"\"\n",
        "    previous_answer = \"\"\n",
        "  else:\n",
        "    previous_topic = current_topic\n",
        "    previous_question = question\n",
        "    previous_answer = answer\n",
        "\n",
        "  temp = \"PreviousT: \" + previous_topic + \"\\nPreviousQ: \" + previous_question + \"\\nPreviousA: \" + previous_answer + \"\\n\\n\"\n",
        "  \n",
        "  current_topic = conversation[\"Topic\"]\n",
        "  question = conversation[\"Question\"]\n",
        "  answer = conversation[\"Answer\"]\n",
        "  prompt = \"Topic: \" + current_topic + \"\\nQuestion: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance_5[\"prompt\"] = temp + prompt\n",
        "  prompt_completetion_instance_5[\"completion\"] = completition\n",
        "\n",
        "\n",
        "\n",
        "  data_prompt_completion_5.append(prompt_completetion_instance_5)"
      ],
      "metadata": {
        "id": "A76h7Om0KpTh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jsonlines"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40ae3129-2697-4fcc-e3c4-d25c4f2bf3ac",
        "id": "llqtMjkcKpTi"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: jsonlines in /usr/local/lib/python3.9/dist-packages (3.1.0)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.9/dist-packages (from jsonlines) (23.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_context_topic_topic = random.sample(data_prompt_completion_5, 5000)\n",
        "\n",
        "with jsonlines.open('gpt3_dev_topic_data_topic_data.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_context_topic_topic)\n",
        "\n",
        "files.download('gpt3_dev_topic_data_topic_data.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "E2C06DjCRNoj",
        "outputId": "6379eebf-7c03-4831-c258-5cfdc3d0ead4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_8e15b1b5-99c7-4f28-b9db-ce9d6cdd71c3\", \"gpt3_dev_topic_data_topic_data.jsonl\", 70220)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# (Question Answer)\n",
        "\n",
        "data_prompt_completion_7 = []\n",
        "\n",
        "for conversation in data:\n",
        "  context = conversation[\"Context\"]\n",
        "  prompt_completetion_instance = {}\n",
        "  prompt = \"\"\n",
        "\n",
        "  question = conversation[\"Question\"]\n",
        "  prompt += \"Question: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "  prompt_completetion_instance[\"prompt\"] = prompt\n",
        "\n",
        "  answer = conversation[\"Answer\"]\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance[\"completion\"] = completition\n",
        "\n",
        "  data_prompt_completion_7.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "puWRDX76GbD7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_prompt_completion_7"
      ],
      "metadata": {
        "id": "rCzq3VPiHLsa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install jsonlines\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_question_answer.jsonl', 'w') as writer:\n",
        "    writer.write_all(data_prompt_completion_7)\n",
        "\n",
        "files.download('gpt3_dev_question_answer.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "ItUDbXvYHCeK",
        "outputId": "5db6ebc7-6acc-4c6c-f59a-6397a602adec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_f13c87bb-36d1-4d94-9569-04776a229857\", \"gpt3_dev_question_answer.jsonl\", 439715)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "small_train_context_topic_topic = random.sample(data_prompt_completion_7, 250)\n",
        "\n",
        "import jsonlines\n",
        "from google.colab import files\n",
        "\n",
        "with jsonlines.open('gpt3_dev_question_answer_trial.jsonl', 'w') as writer:\n",
        "    writer.write_all(small_train_context_topic_topic)\n",
        "\n",
        "files.download('gpt3_dev_question_answer_trial.jsonl')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "0IemaYEFGnvE",
        "outputId": "55e24b05-57dd-468e-ba9b-b5ec3b9488f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_cc81ae53-b620-4f59-b84d-1aab294745f7\", \"gpt3_dev_question_answer_trial.jsonl\", 43713)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_prompt_completion_8 = []\n",
        "previous_topic = \"\"\n",
        "previous_question = \"\"\n",
        "previous_answer = \"\"\n",
        "current_topic = \"\"\n",
        "current_question = \"\"\n",
        "current_answer = \"\"\n",
        "prompt = \"\"\n",
        "for conversation in data:\n",
        "  prompt_completetion_instance = {}\n",
        "\n",
        "\n",
        "  if len(conversation[\"Context\"]) == 0:\n",
        "    previous_topic = \"\"\n",
        "    previous_question = \"\"\n",
        "    previous_answer = \"\"\n",
        "  else:\n",
        "    previous_topic = current_topic\n",
        "    previous_question = current_question\n",
        "    previous_answer = current_answer\n",
        "\n",
        "  for i, qa in enumerate(context):\n",
        "    if i%2 == 0:\n",
        "      prompt += \"Question: \" + qa + \"\\n\"\n",
        "    else:\n",
        "      prompt += \"Answer: \" + qa + \"\\n\"\n",
        "\n",
        "  temp = \"PreviousT: \" + previous_topic + \"\\nPreviousQ: \" + previous_question + \"\\nPreviousA: \" + previous_answer + \"\\n\\n\"\n",
        "  \n",
        "  current_topic = conversation[\"Topic\"]\n",
        "  question = conversation[\"Question\"]\n",
        "  answer = conversation[\"Answer\"]\n",
        "  prompt = \"Topic: \" + current_topic + \"\\nQuestion: \" + question + \"\\nAnswer:\\n\\n###\\n\\n\"\n",
        "\n",
        "\n",
        "  completition = \" \" + answer + \" END\"\n",
        "  prompt_completetion_instance[\"prompt\"] = temp + prompt\n",
        "  prompt_completetion_instance[\"completion\"] = completition\n",
        "\n",
        "\n",
        "\n",
        "  data_prompt_completion_8.append(prompt_completetion_instance)"
      ],
      "metadata": {
        "id": "eAiN-w2duP6i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cpjbU_atHcDE"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}