01_FinRAG_Embedded_Tables_aaron.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "piBrI5ukuZSg"
      },
      "source": [
        "# Chat with Data Notebook (Embedded Tables)\n",
        "\n",
        "In this notebook we walk you through an advanced RAG use case - parsing a complex document with embedded tables - and how to handle that in LlamaIndex.\n",
        "\n",
        "We compare our approach against a \"naive\" RAG stack (using top-k retrieval with a fixed chunk size).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M4aWI5yfuC22",
        "outputId": "590ed473-98e8-4a4c-afa7-86b9100a9433"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting llama-index\n",
            "  Downloading llama_index-0.11.18-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting llama-hub\n",
            "  Downloading llama_hub-0.0.79.post1-py3-none-any.whl.metadata (16 kB)\n",
            "Collecting pypdf\n",
            "  Downloading pypdf-5.0.1-py3-none-any.whl.metadata (7.4 kB)\n",
            "Collecting llama-index-agent-openai<0.4.0,>=0.3.4 (from llama-index)\n",
            "  Downloading llama_index_agent_openai-0.3.4-py3-none-any.whl.metadata (728 bytes)\n",
            "Collecting llama-index-cli<0.4.0,>=0.3.1 (from llama-index)\n",
            "  Downloading llama_index_cli-0.3.1-py3-none-any.whl.metadata (1.5 kB)\n",
            "Collecting llama-index-core<0.12.0,>=0.11.18 (from llama-index)\n",
            "  Downloading llama_index_core-0.11.18-py3-none-any.whl.metadata (2.4 kB)\n",
            "Collecting llama-index-embeddings-openai<0.3.0,>=0.2.4 (from llama-index)\n",
            "  Downloading llama_index_embeddings_openai-0.2.5-py3-none-any.whl.metadata (686 bytes)\n",
            "Collecting llama-index-indices-managed-llama-cloud>=0.3.0 (from llama-index)\n",
            "  Downloading llama_index_indices_managed_llama_cloud-0.4.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "Collecting llama-index-legacy<0.10.0,>=0.9.48 (from llama-index)\n",
            "  Downloading llama_index_legacy-0.9.48.post3-py3-none-any.whl.metadata (8.5 kB)\n",
            "Collecting llama-index-llms-openai<0.3.0,>=0.2.10 (from llama-index)\n",
            "  Downloading llama_index_llms_openai-0.2.15-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting llama-index-multi-modal-llms-openai<0.3.0,>=0.2.0 (from llama-index)\n",
            "  Downloading llama_index_multi_modal_llms_openai-0.2.2-py3-none-any.whl.metadata (678 bytes)\n",
            "Collecting llama-index-program-openai<0.3.0,>=0.2.0 (from llama-index)\n",
            "  Downloading llama_index_program_openai-0.2.0-py3-none-any.whl.metadata (766 bytes)\n",
            "Collecting llama-index-question-gen-openai<0.3.0,>=0.2.0 (from llama-index)\n",
            "  Downloading llama_index_question_gen_openai-0.2.0-py3-none-any.whl.metadata (785 bytes)\n",
            "Collecting llama-index-readers-file<0.3.0,>=0.2.0 (from llama-index)\n",
            "  Downloading llama_index_readers_file-0.2.2-py3-none-any.whl.metadata (5.4 kB)\n",
            "Collecting llama-index-readers-llama-parse>=0.3.0 (from llama-index)\n",
            "  Downloading llama_index_readers_llama_parse-0.3.0-py3-none-any.whl.metadata (3.5 kB)\n",
            "Collecting nltk>3.8.1 (from llama-index)\n",
            "  Downloading nltk-3.9.1-py3-none-any.whl.metadata (2.9 kB)\n",
            "Collecting html2text (from llama-hub)\n",
            "  Downloading html2text-2024.2.26.tar.gz (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from llama-hub) (5.9.5)\n",
            "Collecting pyaml<24.0.0,>=23.9.7 (from llama-hub)\n",
            "  Downloading pyaml-23.12.0-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting retrying (from llama-hub)\n",
            "  Downloading retrying-1.3.4-py3-none-any.whl.metadata (6.9 kB)\n",
            "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.12.2)\n",
            "Collecting openai>=1.14.0 (from llama-index-agent-openai<0.4.0,>=0.3.4->llama-index)\n",
            "  Downloading openai-1.52.0-py3-none-any.whl.metadata (24 kB)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.18->llama-index) (2.0.35)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (3.10.10)\n",
            "Collecting dataclasses-json (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (1.2.14)\n",
            "Collecting dirtyjson<2.0.0,>=1.0.8 (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading dirtyjson-1.0.8-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (2024.6.1)\n",
            "Collecting httpx (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (3.4)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (2.32.3)\n",
            "Collecting tenacity!=8.4.0,<9.0.0,>=8.2.0 (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading tenacity-8.5.0-py3-none-any.whl.metadata (1.2 kB)\n",
            "Collecting tiktoken>=0.3.3 (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (4.66.5)\n",
            "Collecting typing-inspect>=0.8.0 (from llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.18->llama-index) (1.16.0)\n",
            "Collecting llama-cloud>=0.0.11 (from llama-index-indices-managed-llama-cloud>=0.3.0->llama-index)\n",
            "  Downloading llama_cloud-0.1.4-py3-none-any.whl.metadata (814 bytes)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-legacy<0.10.0,>=0.9.48->llama-index) (2.2.2)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file<0.3.0,>=0.2.0->llama-index) (4.12.3)\n",
            "Collecting pypdf\n",
            "  Downloading pypdf-4.3.1-py3-none-any.whl.metadata (7.4 kB)\n",
            "Collecting striprtf<0.0.27,>=0.0.26 (from llama-index-readers-file<0.3.0,>=0.2.0->llama-index)\n",
            "  Downloading striprtf-0.0.26-py3-none-any.whl.metadata (2.1 kB)\n",
            "Collecting llama-parse>=0.5.0 (from llama-index-readers-llama-parse>=0.3.0->llama-index)\n",
            "  Downloading llama_parse-0.5.10-py3-none-any.whl.metadata (6.9 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index) (2024.9.11)\n",
            "Requirement already satisfied: six>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from retrying->llama-hub) (1.16.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (1.14.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (4.0.3)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.3.0,>=0.2.0->llama-index) (2.6)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.18->llama-index) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.18->llama-index) (2024.8.30)\n",
            "Collecting httpcore==1.* (from httpx->llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading httpcore-1.0.6-py3-none-any.whl.metadata (21 kB)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.18->llama-index) (3.10)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.18->llama-index) (1.3.1)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.14.0->llama-index-agent-openai<0.4.0,>=0.3.4->llama-index) (1.7.0)\n",
            "Collecting jiter<1,>=0.4.0 (from openai>=1.14.0->llama-index-agent-openai<0.4.0,>=0.3.4->llama-index)\n",
            "  Downloading jiter-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.2 kB)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.18->llama-index) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.18->llama-index) (2.23.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.18->llama-index) (3.4.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.18->llama-index) (2.2.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.18->llama-index) (3.1.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->llama-index-core<0.12.0,>=0.11.18->llama-index)\n",
            "  Downloading marshmallow-3.23.0-py3-none-any.whl.metadata (7.6 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama-index) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama-index) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-legacy<0.10.0,>=0.9.48->llama-index) (2024.2)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.18->llama-index) (1.2.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.12.0,>=0.11.18->llama-index) (24.1)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.18->llama-index) (0.2.0)\n",
            "Downloading llama_index-0.11.18-py3-none-any.whl (6.8 kB)\n",
            "Downloading llama_hub-0.0.79.post1-py3-none-any.whl (103.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.9/103.9 MB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_agent_openai-0.3.4-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_cli-0.3.1-py3-none-any.whl (27 kB)\n",
            "Downloading llama_index_core-0.11.18-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_embeddings_openai-0.2.5-py3-none-any.whl (6.1 kB)\n",
            "Downloading llama_index_indices_managed_llama_cloud-0.4.0-py3-none-any.whl (10 kB)\n",
            "Downloading llama_index_legacy-0.9.48.post3-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m48.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_llms_openai-0.2.15-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_multi_modal_llms_openai-0.2.2-py3-none-any.whl (5.9 kB)\n",
            "Downloading llama_index_program_openai-0.2.0-py3-none-any.whl (5.3 kB)\n",
            "Downloading llama_index_question_gen_openai-0.2.0-py3-none-any.whl (2.9 kB)\n",
            "Downloading llama_index_readers_file-0.2.2-py3-none-any.whl (38 kB)\n",
            "Downloading pypdf-4.3.1-py3-none-any.whl (295 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.8/295.8 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_index_readers_llama_parse-0.3.0-py3-none-any.whl (2.5 kB)\n",
            "Downloading nltk-3.9.1-py3-none-any.whl (1.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m53.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyaml-23.12.0-py3-none-any.whl (23 kB)\n",
            "Downloading retrying-1.3.4-py3-none-any.whl (11 kB)\n",
            "Downloading dirtyjson-1.0.8-py3-none-any.whl (25 kB)\n",
            "Downloading llama_cloud-0.1.4-py3-none-any.whl (176 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.8/176.8 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpcore-1.0.6-py3-none-any.whl (78 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.0/78.0 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading llama_parse-0.5.10-py3-none-any.whl (12 kB)\n",
            "Downloading openai-1.52.0-py3-none-any.whl (386 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m386.9/386.9 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading striprtf-0.0.26-py3-none-any.whl (6.9 kB)\n",
            "Downloading tenacity-8.5.0-py3-none-any.whl (28 kB)\n",
            "Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m40.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
            "Downloading jiter-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (325 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.2/325.2 kB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading marshmallow-3.23.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.5/49.5 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: html2text\n",
            "  Building wheel for html2text (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for html2text: filename=html2text-2024.2.26-py3-none-any.whl size=33111 sha256=7831d0fcea580d3a2be54ad8884f8fa5f70ba88caf333a561f0575a8b5912ae1\n",
            "  Stored in directory: /root/.cache/pip/wheels/f3/96/6d/a7eba8f80d31cbd188a2787b81514d82fc5ae6943c44777659\n",
            "Successfully built html2text\n",
            "Installing collected packages: striprtf, dirtyjson, tenacity, retrying, pypdf, pyaml, nltk, mypy-extensions, marshmallow, jiter, html2text, h11, typing-inspect, tiktoken, httpcore, httpx, dataclasses-json, openai, llama-index-core, llama-cloud, llama-parse, llama-index-readers-file, llama-index-llms-openai, llama-index-legacy, llama-index-indices-managed-llama-cloud, llama-index-embeddings-openai, llama-index-readers-llama-parse, llama-index-multi-modal-llms-openai, llama-index-cli, llama-index-agent-openai, llama-index-program-openai, llama-index-question-gen-openai, llama-index, llama-hub\n",
            "  Attempting uninstall: tenacity\n",
            "    Found existing installation: tenacity 9.0.0\n",
            "    Uninstalling tenacity-9.0.0:\n",
            "      Successfully uninstalled tenacity-9.0.0\n",
            "  Attempting uninstall: nltk\n",
            "    Found existing installation: nltk 3.8.1\n",
            "    Uninstalling nltk-3.8.1:\n",
            "      Successfully uninstalled nltk-3.8.1\n",
            "Successfully installed dataclasses-json-0.6.7 dirtyjson-1.0.8 h11-0.14.0 html2text-2024.2.26 httpcore-1.0.6 httpx-0.27.2 jiter-0.6.1 llama-cloud-0.1.4 llama-hub-0.0.79.post1 llama-index-0.11.18 llama-index-agent-openai-0.3.4 llama-index-cli-0.3.1 llama-index-core-0.11.18 llama-index-embeddings-openai-0.2.5 llama-index-indices-managed-llama-cloud-0.4.0 llama-index-legacy-0.9.48.post3 llama-index-llms-openai-0.2.15 llama-index-multi-modal-llms-openai-0.2.2 llama-index-program-openai-0.2.0 llama-index-question-gen-openai-0.2.0 llama-index-readers-file-0.2.2 llama-index-readers-llama-parse-0.3.0 llama-parse-0.5.10 marshmallow-3.23.0 mypy-extensions-1.0.0 nltk-3.9.1 openai-1.52.0 pyaml-23.12.0 pypdf-4.3.1 retrying-1.3.4 striprtf-0.0.26 tenacity-8.5.0 tiktoken-0.8.0 typing-inspect-0.9.0\n"
          ]
        }
      ],
      "source": [
        "!pip install llama-index llama-hub pypdf --upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ht4oSN2PvzUJ"
      },
      "outputs": [],
      "source": [
        "# setup OpenAI\n",
        "import openai\n",
        "\n",
        "openai.api_key = \"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k3CGRBjsuLCB"
      },
      "outputs": [],
      "source": [
        "import nest_asyncio\n",
        "\n",
        "nest_asyncio.apply()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VhuXtvSX2mWo"
      },
      "outputs": [],
      "source": [
        "from IPython.display import HTML, display\n",
        "\n",
        "def set_css():\n",
        "  display(HTML('''\n",
        "  <style>\n",
        "    pre {\n",
        "        white-space: pre-wrap;\n",
        "    }\n",
        "  </style>\n",
        "  '''))\n",
        "get_ipython().events.register('pre_run_cell', set_css)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Y0CWFcEv61y"
      },
      "source": [
        "## Complex Document (with Embedded Tables)\n",
        "\n",
        "In this setting we walk through a document that has an embedded table inside of it."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "XVTmmtjp8qvu",
        "outputId": "3cff7e04-1ecf-4272-e57b-7d24c51fde0b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "python3-tk is already the newest version (3.10.8-1~22.04).\n",
            "The following additional packages will be installed:\n",
            "  fonts-droid-fallback fonts-noto-mono fonts-urw-base35 libgs9 libgs9-common libidn12 libijs-0.35\n",
            "  libjbig2dec0 poppler-data\n",
            "Suggested packages:\n",
            "  fonts-noto fonts-freefont-otf | fonts-freefont-ttf fonts-texgyre ghostscript-x poppler-utils\n",
            "  fonts-japanese-mincho | fonts-ipafont-mincho fonts-japanese-gothic | fonts-ipafont-gothic\n",
            "  fonts-arphic-ukai fonts-arphic-uming fonts-nanum\n",
            "The following NEW packages will be installed:\n",
            "  fonts-droid-fallback fonts-noto-mono fonts-urw-base35 ghostscript libgs9 libgs9-common libidn12\n",
            "  libijs-0.35 libjbig2dec0 poppler-data\n",
            "0 upgraded, 10 newly installed, 0 to remove and 49 not upgraded.\n",
            "Need to get 16.7 MB of archives.\n",
            "After this operation, 63.0 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 fonts-droid-fallback all 1:6.0.1r16-1.1build1 [1,805 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 poppler-data all 0.4.11-1 [2,171 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 fonts-noto-mono all 20201225-1build1 [397 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 fonts-urw-base35 all 20200910-1 [6,367 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgs9-common all 9.55.0~dfsg1-0ubuntu5.9 [752 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libidn12 amd64 1.38-4ubuntu1 [60.0 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy/main amd64 libijs-0.35 amd64 0.35-15build2 [16.5 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjbig2dec0 amd64 0.19-3build2 [64.7 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgs9 amd64 9.55.0~dfsg1-0ubuntu5.9 [5,033 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 ghostscript amd64 9.55.0~dfsg1-0ubuntu5.9 [49.5 kB]\n",
            "Fetched 16.7 MB in 3s (5,760 kB/s)\n",
            "Selecting previously unselected package fonts-droid-fallback.\n",
            "(Reading database ... 123629 files and directories currently installed.)\n",
            "Preparing to unpack .../0-fonts-droid-fallback_1%3a6.0.1r16-1.1build1_all.deb ...\n",
            "Unpacking fonts-droid-fallback (1:6.0.1r16-1.1build1) ...\n",
            "Selecting previously unselected package poppler-data.\n",
            "Preparing to unpack .../1-poppler-data_0.4.11-1_all.deb ...\n",
            "Unpacking poppler-data (0.4.11-1) ...\n",
            "Selecting previously unselected package fonts-noto-mono.\n",
            "Preparing to unpack .../2-fonts-noto-mono_20201225-1build1_all.deb ...\n",
            "Unpacking fonts-noto-mono (20201225-1build1) ...\n",
            "Selecting previously unselected package fonts-urw-base35.\n",
            "Preparing to unpack .../3-fonts-urw-base35_20200910-1_all.deb ...\n",
            "Unpacking fonts-urw-base35 (20200910-1) ...\n",
            "Selecting previously unselected package libgs9-common.\n",
            "Preparing to unpack .../4-libgs9-common_9.55.0~dfsg1-0ubuntu5.9_all.deb ...\n",
            "Unpacking libgs9-common (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Selecting previously unselected package libidn12:amd64.\n",
            "Preparing to unpack .../5-libidn12_1.38-4ubuntu1_amd64.deb ...\n",
            "Unpacking libidn12:amd64 (1.38-4ubuntu1) ...\n",
            "Selecting previously unselected package libijs-0.35:amd64.\n",
            "Preparing to unpack .../6-libijs-0.35_0.35-15build2_amd64.deb ...\n",
            "Unpacking libijs-0.35:amd64 (0.35-15build2) ...\n",
            "Selecting previously unselected package libjbig2dec0:amd64.\n",
            "Preparing to unpack .../7-libjbig2dec0_0.19-3build2_amd64.deb ...\n",
            "Unpacking libjbig2dec0:amd64 (0.19-3build2) ...\n",
            "Selecting previously unselected package libgs9:amd64.\n",
            "Preparing to unpack .../8-libgs9_9.55.0~dfsg1-0ubuntu5.9_amd64.deb ...\n",
            "Unpacking libgs9:amd64 (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Selecting previously unselected package ghostscript.\n",
            "Preparing to unpack .../9-ghostscript_9.55.0~dfsg1-0ubuntu5.9_amd64.deb ...\n",
            "Unpacking ghostscript (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Setting up fonts-noto-mono (20201225-1build1) ...\n",
            "Setting up libijs-0.35:amd64 (0.35-15build2) ...\n",
            "Setting up fonts-urw-base35 (20200910-1) ...\n",
            "Setting up poppler-data (0.4.11-1) ...\n",
            "Setting up libjbig2dec0:amd64 (0.19-3build2) ...\n",
            "Setting up libidn12:amd64 (1.38-4ubuntu1) ...\n",
            "Setting up fonts-droid-fallback (1:6.0.1r16-1.1build1) ...\n",
            "Setting up libgs9-common (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Setting up libgs9:amd64 (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Setting up ghostscript (9.55.0~dfsg1-0ubuntu5.9) ...\n",
            "Processing triggers for fontconfig (2.13.1-4.2ubuntu5) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.10.2-1) ...\n"
          ]
        }
      ],
      "source": [
        "!apt install ghostscript python3-tk"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "kv6qso2Bv6ZA",
        "outputId": "b5e1243e-f750-4d95-d2fb-979552ed75cb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting camelot-py\n",
            "  Downloading camelot_py-0.11.0-py3-none-any.whl.metadata (8.3 kB)\n",
            "Collecting pymupdf\n",
            "  Downloading PyMuPDF-1.24.11-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (3.4 kB)\n",
            "Collecting frontend\n",
            "  Downloading frontend-0.0.3-py3-none-any.whl.metadata (847 bytes)\n",
            "Collecting ghostscript\n",
            "  Downloading ghostscript-0.7-py2.py3-none-any.whl.metadata (4.4 kB)\n",
            "Requirement already satisfied: chardet>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (5.2.0)\n",
            "Requirement already satisfied: click>=6.7 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (8.1.7)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (1.26.4)\n",
            "Requirement already satisfied: openpyxl>=2.5.8 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (3.1.5)\n",
            "Requirement already satisfied: pandas>=0.23.4 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (2.2.2)\n",
            "Collecting pdfminer.six>=20200726 (from camelot-py)\n",
            "  Downloading pdfminer.six-20240706-py3-none-any.whl.metadata (4.1 kB)\n",
            "Requirement already satisfied: pypdf>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (4.3.1)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from camelot-py) (0.9.0)\n",
            "Collecting starlette>=0.12.0 (from frontend)\n",
            "  Downloading starlette-0.41.0-py3-none-any.whl.metadata (6.0 kB)\n",
            "Collecting uvicorn>=0.7.1 (from frontend)\n",
            "  Downloading uvicorn-0.32.0-py3-none-any.whl.metadata (6.6 kB)\n",
            "Requirement already satisfied: itsdangerous>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from frontend) (2.2.0)\n",
            "Collecting aiofiles (from frontend)\n",
            "  Downloading aiofiles-24.1.0-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: setuptools>=38.6.0 in /usr/local/lib/python3.10/dist-packages (from ghostscript) (71.0.4)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl>=2.5.8->camelot-py) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=0.23.4->camelot-py) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=0.23.4->camelot-py) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas>=0.23.4->camelot-py) (2024.2)\n",
            "Requirement already satisfied: charset-normalizer>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six>=20200726->camelot-py) (3.4.0)\n",
            "Requirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.10/dist-packages (from pdfminer.six>=20200726->camelot-py) (43.0.1)\n",
            "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf>=3.0.0->camelot-py) (4.12.2)\n",
            "Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette>=0.12.0->frontend) (3.7.1)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn>=0.7.1->frontend) (0.14.0)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette>=0.12.0->frontend) (3.10)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette>=0.12.0->frontend) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette>=0.12.0->frontend) (1.2.2)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=36.0.0->pdfminer.six>=20200726->camelot-py) (1.17.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas>=0.23.4->camelot-py) (1.16.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six>=20200726->camelot-py) (2.22)\n",
            "Downloading camelot_py-0.11.0-py3-none-any.whl (40 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.0/41.0 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading PyMuPDF-1.24.11-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (19.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.6/19.6 MB\u001b[0m \u001b[31m97.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading frontend-0.0.3-py3-none-any.whl (32 kB)\n",
            "Downloading ghostscript-0.7-py2.py3-none-any.whl (25 kB)\n",
            "Downloading pdfminer.six-20240706-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m108.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading starlette-0.41.0-py3-none-any.whl (73 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.2/73.2 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading uvicorn-0.32.0-py3-none-any.whl (63 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.7/63.7 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading aiofiles-24.1.0-py3-none-any.whl (15 kB)\n",
            "Installing collected packages: uvicorn, pymupdf, ghostscript, aiofiles, starlette, pdfminer.six, frontend, camelot-py\n",
            "Successfully installed aiofiles-24.1.0 camelot-py-0.11.0 frontend-0.0.3 ghostscript-0.7 pdfminer.six-20240706 pymupdf-1.24.11 starlette-0.41.0 uvicorn-0.32.0\n"
          ]
        }
      ],
      "source": [
        "!pip install camelot-py pymupdf frontend ghostscript"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252
        },
        "id": "_cJgsxfOB5Ch",
        "outputId": "36778e0c-257a-4393-b5c3-2f25105e4b54"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: matplotlib-venn in /usr/local/lib/python3.10/dist-packages (1.1.1)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from matplotlib-venn) (3.7.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from matplotlib-venn) (1.26.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from matplotlib-venn) (1.13.1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (1.4.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (24.1)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (10.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (3.1.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->matplotlib-venn) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->matplotlib-venn) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "# aaron 1\n",
        "!pip install matplotlib-venn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "jVu3LoqTENCZ",
        "outputId": "49188f49-fce3-40ff-db75-cd3a8ccf63ac"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rHit:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Hit:3 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n",
            "Hit:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Ign:9 https://r2u.stat.illinois.edu/ubuntu jammy InRelease\n",
            "Hit:10 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Hit:11 https://r2u.stat.illinois.edu/ubuntu jammy Release\n",
            "Reading package lists... Done\n",
            "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n"
          ]
        }
      ],
      "source": [
        "!apt-get update"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "dj9L_qSxERhL",
        "outputId": "cf43f1fd-9c9a-4ffb-9e9a-fc132fc73c0d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rHit:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n",
            "Hit:6 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Hit:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:8 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Ign:9 https://r2u.stat.illinois.edu/ubuntu jammy InRelease\n",
            "Hit:10 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Hit:11 https://r2u.stat.illinois.edu/ubuntu jammy Release\n",
            "Reading package lists... Done\n",
            "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n"
          ]
        }
      ],
      "source": [
        "!apt-get update --fix-missing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "id": "bjGkFjZtEX1g",
        "outputId": "217d69eb-da0a-420b-8c27-87eb2573906c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Calculating upgrade... Done\n",
            "The following packages were automatically installed and are no longer required:\n",
            "  linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic\n",
            "Use 'apt autoremove' to remove them.\n",
            "The following packages have been kept back:\n",
            "  libcudnn8 libcudnn8-dev libnccl-dev libnccl2\n",
            "0 upgraded, 0 newly installed, 0 to remove and 4 not upgraded.\n"
          ]
        }
      ],
      "source": [
        "!apt-get upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "wBDz3JX_CCKR",
        "outputId": "1780c4cb-bd78-4c3c-c335-e20516f96149"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "E: Package 'libfluidsynth1' has no installation candidate\n"
          ]
        }
      ],
      "source": [
        "# aaron 2\n",
        "!apt-get -qq install -y libfluidsynth1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "45QwTBX_CYuW",
        "outputId": "2589e78c-8310-44c3-dd4d-3da13506d101"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "E: Unable to locate package update\n"
          ]
        }
      ],
      "source": [
        "# aaron\n",
        "!apt-get install update"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "pAtV1zdlCTdz",
        "outputId": "db7c8a12-9fde-4256-d54c-01e49d4a7b44"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "E: Unable to locate package upgrade\n"
          ]
        }
      ],
      "source": [
        "# aaron\n",
        "!apt-get install upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 506
        },
        "id": "ImBOlgQPDsLp",
        "outputId": "28951bec-0b1c-49df-d560-dc0062cc862b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease [1,581 B]\n",
            "Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  Packages [1,032 kB]\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
            "Get:6 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
            "Ign:7 https://r2u.stat.illinois.edu/ubuntu jammy InRelease\n",
            "Get:8 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease [18.1 kB]\n",
            "Get:9 https://r2u.stat.illinois.edu/ubuntu jammy Release [5,713 B]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]\n",
            "Hit:11 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Get:12 https://r2u.stat.illinois.edu/ubuntu jammy Release.gpg [793 B]\n",
            "Hit:13 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [3,278 kB]\n",
            "Get:15 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages [33.9 kB]\n",
            "Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [3,200 kB]\n",
            "Get:17 https://r2u.stat.illinois.edu/ubuntu jammy/main amd64 Packages [2,600 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,648 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,451 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [33.7 kB]\n",
            "Get:21 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,162 kB]\n",
            "Get:22 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2,372 kB]\n",
            "Get:23 https://r2u.stat.illinois.edu/ubuntu jammy/main all Packages [8,405 kB]\n",
            "Fetched 26.6 MB in 5s (5,710 kB/s)\n",
            "Reading package lists... Done\n",
            "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n"
          ]
        }
      ],
      "source": [
        "# aaron\n",
        "!apt-get update"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Iwggep1UDyGq",
        "outputId": "1704e03e-c569-4e0f-b0f3-73925b947069"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Calculating upgrade... Done\n",
            "The following packages were automatically installed and are no longer required:\n",
            "  linux-headers-5.15.0-122 linux-headers-5.15.0-122-generic\n",
            "Use 'apt autoremove' to remove them.\n",
            "The following NEW packages will be installed:\n",
            "  linux-headers-5.15.0-124 linux-headers-5.15.0-124-generic\n",
            "The following packages have been kept back:\n",
            "  libcudnn8 libcudnn8-dev libnccl-dev libnccl2\n",
            "The following packages will be upgraded:\n",
            "  base-files bash binutils binutils-common binutils-x86-64-linux-gnu bsdutils coreutils\n",
            "  cuda-compat-12-2 cuda-keyring cuda-toolkit-12-config-common cuda-toolkit-config-common dpkg\n",
            "  dpkg-dev e2fsprogs libarchive13 libbinutils libblkid1 libc-bin libctf-nobfd0 libctf0 libdpkg-perl\n",
            "  libext2fs2 libgnutls30 libldap-2.5-0 libmount1 libpam-modules libpam-modules-bin libpam-runtime\n",
            "  libpam0g libperl5.34 libprocps8 libsmartcols1 libss2 libudev1 linux-headers-generic\n",
            "  linux-libc-dev login logsave mount openssl passwd perl perl-base perl-modules-5.34 procps\n",
            "  r-cran-gert r-cran-rstudioapi tar util-linux\n",
            "49 upgraded, 2 newly installed, 0 to remove and 4 not upgraded.\n",
            "Need to get 78.8 MB of archives.\n",
            "After this operation, 103 MB of additional disk space will be used.\n",
            "Get:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-compat-12-2 535.183.06-1 [33.6 MB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 base-files amd64 12ubuntu4.7 [61.9 kB]\n",
            "Get:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-keyring 1.1-1 [4,332 B]\n",
            "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-toolkit-12-config-common 12.6.77-1 [16.5 kB]\n",
            "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  cuda-toolkit-config-common 12.6.77-1 [16.5 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 bash amd64 5.1-6ubuntu1.1 [769 kB]\n",
            "Get:7 https://r2u.stat.illinois.edu/ubuntu jammy/main all r-cran-rstudioapi all 0.17.0-1.ca2204.1 [309 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 bsdutils amd64 1:2.37.2-4ubuntu3.4 [80.9 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 coreutils amd64 8.32-4.1ubuntu1.2 [1,437 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 tar amd64 1.34+dfsg-1ubuntu0.1.22.04.2 [295 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dpkg amd64 1.21.1ubuntu2.3 [1,239 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 login amd64 1:4.8.1-2ubuntu2.2 [188 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libperl5.34 amd64 5.34.0-3ubuntu1.3 [4,820 kB]\n",
            "Get:14 https://r2u.stat.illinois.edu/ubuntu jammy/main amd64 r-cran-gert amd64 2.1.4-1.ca2204.1 [2,516 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl amd64 5.34.0-3ubuntu1.3 [232 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl-base amd64 5.34.0-3ubuntu1.3 [1,762 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 perl-modules-5.34 all 5.34.0-3ubuntu1.3 [2,976 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 util-linux amd64 2.37.2-4ubuntu3.4 [1,063 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libc-bin amd64 2.35-0ubuntu3.8 [706 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam0g amd64 1.4.0-11ubuntu2.4 [60.2 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-modules-bin amd64 1.4.0-11ubuntu2.4 [37.6 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-modules amd64 1.4.0-11ubuntu2.4 [280 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 logsave amd64 1.46.5-2ubuntu1.2 [10.1 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libext2fs2 amd64 1.46.5-2ubuntu1.2 [208 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 e2fsprogs amd64 1.46.5-2ubuntu1.2 [590 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 mount amd64 2.37.2-4ubuntu3.4 [114 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libblkid1 amd64 2.37.2-4ubuntu3.4 [103 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libmount1 amd64 2.37.2-4ubuntu3.4 [122 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpam-runtime all 1.4.0-11ubuntu2.4 [40.3 kB]\n",
            "Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libsmartcols1 amd64 2.37.2-4ubuntu3.4 [50.9 kB]\n",
            "Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.12 [78.2 kB]\n",
            "Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 passwd amd64 1:4.8.1-2ubuntu2.2 [768 kB]\n",
            "Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libgnutls30 amd64 3.7.3-4ubuntu1.5 [966 kB]\n",
            "Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libprocps8 amd64 2:3.3.17-6ubuntu2.1 [36.2 kB]\n",
            "Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libss2 amd64 1.46.5-2ubuntu1.2 [12.3 kB]\n",
            "Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 procps amd64 2:3.3.17-6ubuntu2.1 [378 kB]\n",
            "Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 openssl amd64 3.0.2-0ubuntu1.18 [1,184 kB]\n",
            "Get:38 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf0 amd64 2.38-4ubuntu2.6 [103 kB]\n",
            "Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libctf-nobfd0 amd64 2.38-4ubuntu2.6 [108 kB]\n",
            "Get:40 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-x86-64-linux-gnu amd64 2.38-4ubuntu2.6 [2,326 kB]\n",
            "Get:41 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libbinutils amd64 2.38-4ubuntu2.6 [662 kB]\n",
            "Get:42 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils amd64 2.38-4ubuntu2.6 [3,200 B]\n",
            "Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 binutils-common amd64 2.38-4ubuntu2.6 [222 kB]\n",
            "Get:44 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 dpkg-dev all 1.21.1ubuntu2.3 [922 kB]\n",
            "Get:45 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libdpkg-perl all 1.21.1ubuntu2.3 [237 kB]\n",
            "Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libarchive13 amd64 3.6.0-1ubuntu1.2 [369 kB]\n",
            "Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libldap-2.5-0 amd64 2.5.18+dfsg-0ubuntu0.22.04.2 [183 kB]\n",
            "Get:48 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-124 all 5.15.0-124.134 [12.3 MB]\n",
            "Get:49 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-124-generic amd64 5.15.0-124.134 [2,827 kB]\n",
            "Get:50 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.124.124 [2,274 B]\n",
            "Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-libc-dev amd64 5.15.0-124.134 [1,322 kB]\n",
            "Fetched 78.8 MB in 4s (18.1 MB/s)\n",
            "Extracting templates from packages: 100%\n",
            "Preconfiguring packages ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../base-files_12ubuntu4.7_amd64.deb ...\n",
            "Unpacking base-files (12ubuntu4.7) over (12ubuntu4.4) ...\n",
            "Setting up base-files (12ubuntu4.7) ...\n",
            "Installing new version of config file /etc/issue ...\n",
            "Installing new version of config file /etc/issue.net ...\n",
            "Installing new version of config file /etc/lsb-release ...\n",
            "Installing new version of config file /etc/update-motd.d/10-help-text ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../bash_5.1-6ubuntu1.1_amd64.deb ...\n",
            "Unpacking bash (5.1-6ubuntu1.1) over (5.1-6ubuntu1) ...\n",
            "Setting up bash (5.1-6ubuntu1.1) ...\n",
            "update-alternatives: using /usr/share/man/man7/bash-builtins.7.gz to provide /usr/share/man/man7/builtins.7.gz (builtins.7.gz) in auto mode\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../bsdutils_1%3a2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking bsdutils (1:2.37.2-4ubuntu3.4) over (1:2.37.2-4ubuntu3) ...\n",
            "Setting up bsdutils (1:2.37.2-4ubuntu3.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../coreutils_8.32-4.1ubuntu1.2_amd64.deb ...\n",
            "Unpacking coreutils (8.32-4.1ubuntu1.2) over (8.32-4.1ubuntu1) ...\n",
            "Setting up coreutils (8.32-4.1ubuntu1.2) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../tar_1.34+dfsg-1ubuntu0.1.22.04.2_amd64.deb ...\n",
            "Unpacking tar (1.34+dfsg-1ubuntu0.1.22.04.2) over (1.34+dfsg-1ubuntu0.1.22.04.1) ...\n",
            "Setting up tar (1.34+dfsg-1ubuntu0.1.22.04.2) ...\n",
            "update-alternatives: warning: forcing reinstallation of alternative /usr/sbin/rmt-tar because link group rmt is broken\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../dpkg_1.21.1ubuntu2.3_amd64.deb ...\n",
            "Unpacking dpkg (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Setting up dpkg (1.21.1ubuntu2.3) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../login_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking login (1:4.8.1-2ubuntu2.2) over (1:4.8.1-2ubuntu2.1) ...\n",
            "Setting up login (1:4.8.1-2ubuntu2.2) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libperl5.34_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking libperl5.34:amd64 (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../perl_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking perl (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../perl-base_5.34.0-3ubuntu1.3_amd64.deb ...\n",
            "Unpacking perl-base (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Setting up perl-base (5.34.0-3ubuntu1.3) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../perl-modules-5.34_5.34.0-3ubuntu1.3_all.deb ...\n",
            "Unpacking perl-modules-5.34 (5.34.0-3ubuntu1.3) over (5.34.0-3ubuntu1.2) ...\n",
            "Preparing to unpack .../util-linux_2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking util-linux (2.37.2-4ubuntu3.4) over (2.37.2-4ubuntu3) ...\n",
            "Setting up util-linux (2.37.2-4ubuntu3.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libc-bin_2.35-0ubuntu3.8_amd64.deb ...\n",
            "Unpacking libc-bin (2.35-0ubuntu3.8) over (2.35-0ubuntu3.4) ...\n",
            "Setting up libc-bin (2.35-0ubuntu3.8) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n",
            "\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam0g_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "Unpacking libpam0g:amd64 (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam0g:amd64 (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-modules-bin_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "Unpacking libpam-modules-bin (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-modules-bin (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-modules_1.4.0-11ubuntu2.4_amd64.deb ...\n",
            "Unpacking libpam-modules:amd64 (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-modules:amd64 (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../logsave_1.46.5-2ubuntu1.2_amd64.deb ...\n",
            "Unpacking logsave (1.46.5-2ubuntu1.2) over (1.46.5-2ubuntu1.1) ...\n",
            "Preparing to unpack .../libext2fs2_1.46.5-2ubuntu1.2_amd64.deb ...\n",
            "Unpacking libext2fs2:amd64 (1.46.5-2ubuntu1.2) over (1.46.5-2ubuntu1.1) ...\n",
            "Setting up libext2fs2:amd64 (1.46.5-2ubuntu1.2) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../e2fsprogs_1.46.5-2ubuntu1.2_amd64.deb ...\n",
            "Unpacking e2fsprogs (1.46.5-2ubuntu1.2) over (1.46.5-2ubuntu1.1) ...\n",
            "Preparing to unpack .../mount_2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking mount (2.37.2-4ubuntu3.4) over (2.37.2-4ubuntu3) ...\n",
            "Preparing to unpack .../libblkid1_2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking libblkid1:amd64 (2.37.2-4ubuntu3.4) over (2.37.2-4ubuntu3) ...\n",
            "Setting up libblkid1:amd64 (2.37.2-4ubuntu3.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libmount1_2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking libmount1:amd64 (2.37.2-4ubuntu3.4) over (2.37.2-4ubuntu3) ...\n",
            "Setting up libmount1:amd64 (2.37.2-4ubuntu3.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libpam-runtime_1.4.0-11ubuntu2.4_all.deb ...\n",
            "Unpacking libpam-runtime (1.4.0-11ubuntu2.4) over (1.4.0-11ubuntu2.3) ...\n",
            "Setting up libpam-runtime (1.4.0-11ubuntu2.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libsmartcols1_2.37.2-4ubuntu3.4_amd64.deb ...\n",
            "Unpacking libsmartcols1:amd64 (2.37.2-4ubuntu3.4) over (2.37.2-4ubuntu3) ...\n",
            "Setting up libsmartcols1:amd64 (2.37.2-4ubuntu3.4) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libudev1_249.11-0ubuntu3.12_amd64.deb ...\n",
            "Unpacking libudev1:amd64 (249.11-0ubuntu3.12) over (249.11-0ubuntu3.10) ...\n",
            "Setting up libudev1:amd64 (249.11-0ubuntu3.12) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../passwd_1%3a4.8.1-2ubuntu2.2_amd64.deb ...\n",
            "Unpacking passwd (1:4.8.1-2ubuntu2.2) over (1:4.8.1-2ubuntu2.1) ...\n",
            "Setting up passwd (1:4.8.1-2ubuntu2.2) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../libgnutls30_3.7.3-4ubuntu1.5_amd64.deb ...\n",
            "Unpacking libgnutls30:amd64 (3.7.3-4ubuntu1.5) over (3.7.3-4ubuntu1.2) ...\n",
            "Setting up libgnutls30:amd64 (3.7.3-4ubuntu1.5) ...\n",
            "(Reading database ... 124732 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libprocps8_2%3a3.3.17-6ubuntu2.1_amd64.deb ...\n",
            "Unpacking libprocps8:amd64 (2:3.3.17-6ubuntu2.1) over (2:3.3.17-6ubuntu2) ...\n",
            "Preparing to unpack .../01-libss2_1.46.5-2ubuntu1.2_amd64.deb ...\n",
            "Unpacking libss2:amd64 (1.46.5-2ubuntu1.2) over (1.46.5-2ubuntu1.1) ...\n",
            "Preparing to unpack .../02-procps_2%3a3.3.17-6ubuntu2.1_amd64.deb ...\n",
            "Unpacking procps (2:3.3.17-6ubuntu2.1) over (2:3.3.17-6ubuntu2) ...\n",
            "Preparing to unpack .../03-openssl_3.0.2-0ubuntu1.18_amd64.deb ...\n",
            "Unpacking openssl (3.0.2-0ubuntu1.18) over (3.0.2-0ubuntu1.12) ...\n",
            "Preparing to unpack .../04-libctf0_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libctf0:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../05-libctf-nobfd0_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libctf-nobfd0:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../06-binutils-x86-64-linux-gnu_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils-x86-64-linux-gnu (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../07-libbinutils_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking libbinutils:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../08-binutils_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../09-binutils-common_2.38-4ubuntu2.6_amd64.deb ...\n",
            "Unpacking binutils-common:amd64 (2.38-4ubuntu2.6) over (2.38-4ubuntu2.3) ...\n",
            "Preparing to unpack .../10-cuda-compat-12-2_535.183.06-1_amd64.deb ...\n",
            "Unpacking cuda-compat-12-2 (535.183.06-1) over (535.129.03-1) ...\n",
            "Preparing to unpack .../11-cuda-keyring_1.1-1_all.deb ...\n",
            "Unpacking cuda-keyring (1.1-1) over (1.0-1) ...\n",
            "Preparing to unpack .../12-cuda-toolkit-12-config-common_12.6.77-1_all.deb ...\n",
            "Unpacking cuda-toolkit-12-config-common (12.6.77-1) over (12.3.52-1) ...\n",
            "Preparing to unpack .../13-cuda-toolkit-config-common_12.6.77-1_all.deb ...\n",
            "Unpacking cuda-toolkit-config-common (12.6.77-1) over (12.3.52-1) ...\n",
            "Preparing to unpack .../14-dpkg-dev_1.21.1ubuntu2.3_all.deb ...\n",
            "Unpacking dpkg-dev (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Preparing to unpack .../15-libdpkg-perl_1.21.1ubuntu2.3_all.deb ...\n",
            "Unpacking libdpkg-perl (1.21.1ubuntu2.3) over (1.21.1ubuntu2.2) ...\n",
            "Preparing to unpack .../16-libarchive13_3.6.0-1ubuntu1.2_amd64.deb ...\n",
            "Unpacking libarchive13:amd64 (3.6.0-1ubuntu1.2) over (3.6.0-1ubuntu1.1) ...\n",
            "Preparing to unpack .../17-libldap-2.5-0_2.5.18+dfsg-0ubuntu0.22.04.2_amd64.deb ...\n",
            "Unpacking libldap-2.5-0:amd64 (2.5.18+dfsg-0ubuntu0.22.04.2) over (2.5.16+dfsg-0ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package linux-headers-5.15.0-124.\n",
            "Preparing to unpack .../18-linux-headers-5.15.0-124_5.15.0-124.134_all.deb ...\n",
            "Unpacking linux-headers-5.15.0-124 (5.15.0-124.134) ...\n",
            "Selecting previously unselected package linux-headers-5.15.0-124-generic.\n",
            "Preparing to unpack .../19-linux-headers-5.15.0-124-generic_5.15.0-124.134_amd64.deb ...\n",
            "Unpacking linux-headers-5.15.0-124-generic (5.15.0-124.134) ...\n",
            "Preparing to unpack .../20-linux-headers-generic_5.15.0.124.124_amd64.deb ...\n",
            "Unpacking linux-headers-generic (5.15.0.124.124) over (5.15.0.122.122) ...\n",
            "Preparing to unpack .../21-linux-libc-dev_5.15.0-124.134_amd64.deb ...\n",
            "Unpacking linux-libc-dev:amd64 (5.15.0-124.134) over (5.15.0-88.98) ...\n",
            "Preparing to unpack .../22-r-cran-rstudioapi_0.17.0-1.ca2204.1_all.deb ...\n",
            "Unpacking r-cran-rstudioapi (0.17.0-1.ca2204.1) over (0.16.0-1.ca2204.1) ...\n",
            "Preparing to unpack .../23-r-cran-gert_2.1.4-1.ca2204.1_amd64.deb ...\n",
            "Unpacking r-cran-gert (2.1.4-1.ca2204.1) over (2.1.2-1.ca2204.1) ...\n",
            "Setting up cuda-toolkit-config-common (12.6.77-1) ...\n",
            "Setting up cuda-compat-12-2 (535.183.06-1) ...\n",
            "Setting up libarchive13:amd64 (3.6.0-1ubuntu1.2) ...\n",
            "Setting up linux-headers-5.15.0-124 (5.15.0-124.134) ...\n",
            "Setting up binutils-common:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up linux-libc-dev:amd64 (5.15.0-124.134) ...\n",
            "Setting up libctf-nobfd0:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up perl-modules-5.34 (5.34.0-3ubuntu1.3) ...\n",
            "Setting up libldap-2.5-0:amd64 (2.5.18+dfsg-0ubuntu0.22.04.2) ...\n",
            "Setting up r-cran-rstudioapi (0.17.0-1.ca2204.1) ...\n",
            "Setting up cuda-keyring (1.1-1) ...\n",
            "Setting up libss2:amd64 (1.46.5-2ubuntu1.2) ...\n",
            "Setting up logsave (1.46.5-2ubuntu1.2) ...\n",
            "Setting up linux-headers-5.15.0-124-generic (5.15.0-124.134) ...\n",
            "Setting up mount (2.37.2-4ubuntu3.4) ...\n",
            "Setting up libbinutils:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up openssl (3.0.2-0ubuntu1.18) ...\n",
            "Setting up cuda-toolkit-12-config-common (12.6.77-1) ...\n",
            "Setting up libprocps8:amd64 (2:3.3.17-6ubuntu2.1) ...\n",
            "Setting up libctf0:amd64 (2.38-4ubuntu2.6) ...\n",
            "Setting up libperl5.34:amd64 (5.34.0-3ubuntu1.3) ...\n",
            "Setting up r-cran-gert (2.1.4-1.ca2204.1) ...\n",
            "Setting up e2fsprogs (1.46.5-2ubuntu1.2) ...\n",
            "Setting up linux-headers-generic (5.15.0.124.124) ...\n",
            "Setting up perl (5.34.0-3ubuntu1.3) ...\n",
            "Setting up libdpkg-perl (1.21.1ubuntu2.3) ...\n",
            "Setting up procps (2:3.3.17-6ubuntu2.1) ...\n",
            "Setting up binutils-x86-64-linux-gnu (2.38-4ubuntu2.6) ...\n",
            "Setting up binutils (2.38-4ubuntu2.6) ...\n",
            "Setting up dpkg-dev (1.21.1ubuntu2.3) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.8) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!apt upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 688
        },
        "id": "VzCDzgRvFS37",
        "outputId": "95f206cb-bb81-40cd-ee77-ab5189dea60e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting libarchive\n",
            "  Using cached libarchive-0.4.7.tar.gz (23 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting nose (from libarchive)\n",
            "  Using cached nose-1.3.7-py3-none-any.whl.metadata (1.7 kB)\n",
            "Using cached nose-1.3.7-py3-none-any.whl (154 kB)\n",
            "Building wheels for collected packages: libarchive\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py bdist_wheel\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for libarchive (setup.py) ... \u001b[?25lerror\n",
            "\u001b[31m  ERROR: Failed building wheel for libarchive\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[?25h  Running setup.py clean for libarchive\n",
            "Failed to build libarchive\n",
            "\u001b[31mERROR: ERROR: Failed to build installable wheels for some pyproject.toml based projects (libarchive)\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        },
        {
          "ename": "ModuleNotFoundError",
          "evalue": "No module named 'libarchive'",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-31-5bee6841787f>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# https://pypi.python.org/pypi/libarchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'apt-get -qq install -y libarchive-dev && pip install -U libarchive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mlibarchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'libarchive'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo install libarchive, click the button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# https://pypi.python.org/pypi/libarchive\n",
        "!apt-get -qq install -y libarchive-dev && pip install -U libarchive\n",
        "import libarchive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4OTPWLUrFdwE"
      },
      "outputs": [],
      "source": [
        "# https://pypi.python.org/pypi/libarchive\n",
        "!apt-get -qq install -y libarchive-dev && pip install -U libarchive\n",
        "import libarchive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 688
        },
        "id": "oDdPl0lNCIOt",
        "outputId": "9f7acc9a-a2fc-429d-d970-f7fc20dd4aef"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting libarchive\n",
            "  Using cached libarchive-0.4.7.tar.gz (23 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting nose (from libarchive)\n",
            "  Using cached nose-1.3.7-py3-none-any.whl.metadata (1.7 kB)\n",
            "Using cached nose-1.3.7-py3-none-any.whl (154 kB)\n",
            "Building wheels for collected packages: libarchive\n",
            "  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n",
            "  \n",
            "  \u001b[31m×\u001b[0m \u001b[32mpython setup.py bdist_wheel\u001b[0m did not run successfully.\n",
            "  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n",
            "  \u001b[31m╰─>\u001b[0m See above for output.\n",
            "  \n",
            "  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n",
            "  Building wheel for libarchive (setup.py) ... \u001b[?25lerror\n",
            "\u001b[31m  ERROR: Failed building wheel for libarchive\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[?25h  Running setup.py clean for libarchive\n",
            "Failed to build libarchive\n",
            "\u001b[31mERROR: ERROR: Failed to build installable wheels for some pyproject.toml based projects (libarchive)\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        },
        {
          "ename": "ModuleNotFoundError",
          "evalue": "No module named 'libarchive'",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-1c7421369c66>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# https://pypi.python.org/pypi/libarchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'apt-get -qq install -y libarchive-dev && pip install -U libarchive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mlibarchive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'libarchive'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo install libarchive, click the button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# aaron 3\n",
        "# https://pypi.python.org/pypi/libarchive\n",
        "!apt-get -qq install -y libarchive-dev && pip install -U libarchive\n",
        "import libarchive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "47rSalDFCkHG",
        "outputId": "d75f23d5-11e8-44fc-fa05-07c88f78229a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pydot in /usr/local/lib/python3.10/dist-packages (3.0.2)\n",
            "Requirement already satisfied: pyparsing>=3.0.9 in /usr/local/lib/python3.10/dist-packages (from pydot) (3.1.4)\n"
          ]
        }
      ],
      "source": [
        "# aaron 4\n",
        "# https://pypi.python.org/pypi/pydot\n",
        "!apt-get -qq install -y graphviz && pip install pydot\n",
        "import pydot"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 307
        },
        "id": "MdHv39JMCrVR",
        "outputId": "23d2f485-5a7c-4b16-b1d9-6658e4f6e4de"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: cartopy in /usr/local/lib/python3.10/dist-packages (0.24.1)\n",
            "Requirement already satisfied: numpy>=1.23 in /usr/local/lib/python3.10/dist-packages (from cartopy) (1.26.4)\n",
            "Requirement already satisfied: matplotlib>=3.6 in /usr/local/lib/python3.10/dist-packages (from cartopy) (3.7.1)\n",
            "Requirement already satisfied: shapely>=1.8 in /usr/local/lib/python3.10/dist-packages (from cartopy) (2.0.6)\n",
            "Requirement already satisfied: packaging>=21 in /usr/local/lib/python3.10/dist-packages (from cartopy) (24.1)\n",
            "Requirement already satisfied: pyshp>=2.3 in /usr/local/lib/python3.10/dist-packages (from cartopy) (2.3.1)\n",
            "Requirement already satisfied: pyproj>=3.3.1 in /usr/local/lib/python3.10/dist-packages (from cartopy) (3.7.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (1.4.7)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (10.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (3.1.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.6->cartopy) (2.8.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pyproj>=3.3.1->cartopy) (2024.8.30)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.6->cartopy) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "# aaron 5\n",
        "!pip install cartopy\n",
        "import cartopy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aQb2AJNZGn0O",
        "outputId": "fa579a37-08d0-44a6-ab18-4c710e82455a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: llama-index-embeddings-openai in /usr/local/lib/python3.10/dist-packages (0.2.5)\n",
            "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-openai) (0.11.18)\n",
            "Requirement already satisfied: openai>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-openai) (1.52.0)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.0.35)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.10.10)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2024.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.27.2)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.4)\n",
            "Requirement already satisfied: nltk>3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.9.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.32.3)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.8.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (4.66.5)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (4.12.2)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.16.0)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-embeddings-openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.1.0->llama-index-embeddings-openai) (1.7.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-embeddings-openai) (0.6.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.1.0->llama-index-embeddings-openai) (1.3.1)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.14.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (4.0.3)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.1.0->llama-index-embeddings-openai) (3.10)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.1.0->llama-index-embeddings-openai) (1.2.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2024.8.30)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.0.6)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.14.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2024.9.11)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.23.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.4.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (2.2.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.1.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (3.23.0)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (24.1)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-embeddings-openai) (0.2.0)\n",
            "Requirement already satisfied: llama-index-readers-file in /usr/local/lib/python3.10/dist-packages (0.2.2)\n",
            "Requirement already satisfied: pymupdf in /usr/local/lib/python3.10/dist-packages (1.24.11)\n",
            "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file) (4.12.3)\n",
            "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file) (0.11.18)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file) (2.2.2)\n",
            "Requirement already satisfied: pypdf<5.0.0,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file) (4.3.1)\n",
            "Requirement already satisfied: striprtf<0.0.27,>=0.0.26 in /usr/local/lib/python3.10/dist-packages (from llama-index-readers-file) (0.0.26)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file) (2.6)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.0.35)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.10.10)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2024.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.27.2)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.4)\n",
            "Requirement already satisfied: nltk>3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.9.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.32.3)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.8.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (4.66.5)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (4.12.2)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-readers-file) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-readers-file) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-readers-file) (2024.2)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.14.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (4.0.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2024.9.11)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.23.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-index-readers-file) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (2024.8.30)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.1.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.23.0)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.0.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.14.0)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (24.1)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (0.2.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-readers-file) (1.2.2)\n",
            "Requirement already satisfied: llama-index-llms-openai in /usr/local/lib/python3.10/dist-packages (0.2.15)\n",
            "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.7 in /usr/local/lib/python3.10/dist-packages (from llama-index-llms-openai) (0.11.18)\n",
            "Requirement already satisfied: openai<2.0.0,>=1.40.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-llms-openai) (1.52.0)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.0.35)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.10.10)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2024.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.27.2)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.4)\n",
            "Requirement already satisfied: nltk>3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.9.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.32.3)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.8.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (4.66.5)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (4.12.2)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.16.0)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai) (1.7.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai) (0.6.1)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai) (1.3.1)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.14.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (4.0.3)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.40.0->llama-index-llms-openai) (3.10)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.40.0->llama-index-llms-openai) (1.2.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2024.8.30)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.0.6)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.14.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2024.9.11)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.23.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.4.0)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (2.2.3)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.1.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (3.23.0)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (24.1)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.7->llama-index-llms-openai) (0.2.0)\n",
            "Collecting llama-index-experimental\n",
            "  Downloading llama_index_experimental-0.4.0-py3-none-any.whl.metadata (884 bytes)\n",
            "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-experimental) (0.11.18)\n",
            "Collecting llama-index-finetuning<0.3.0,>=0.2.0 (from llama-index-experimental)\n",
            "  Downloading llama_index_finetuning-0.2.1-py3-none-any.whl.metadata (992 bytes)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-index-experimental) (2.2.2)\n",
            "Requirement already satisfied: PyYAML>=6.0.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (6.0.2)\n",
            "Requirement already satisfied: SQLAlchemy>=1.4.49 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.0.35)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.10.10)\n",
            "Requirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.6.7)\n",
            "Requirement already satisfied: deprecated>=1.2.9.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.2.14)\n",
            "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.0.8)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2024.6.1)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.27.2)\n",
            "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.6.0)\n",
            "Requirement already satisfied: networkx>=3.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.4)\n",
            "Requirement already satisfied: nltk>3.8.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.9.1)\n",
            "Requirement already satisfied: numpy<2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.26.4)\n",
            "Requirement already satisfied: pillow>=9.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (10.4.0)\n",
            "Requirement already satisfied: pydantic<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.9.2)\n",
            "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.32.3)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (8.5.0)\n",
            "Requirement already satisfied: tiktoken>=0.3.3 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.8.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.66.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (4.66.5)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (4.12.2)\n",
            "Requirement already satisfied: typing-inspect>=0.8.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.9.0)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.16.0)\n",
            "Collecting llama-index-embeddings-adapter<0.3.0,>=0.2.0 (from llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading llama_index_embeddings_adapter-0.2.2-py3-none-any.whl.metadata (688 bytes)\n",
            "Collecting llama-index-llms-azure-openai<0.3.0,>=0.2.0 (from llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading llama_index_llms_azure_openai-0.2.2-py3-none-any.whl.metadata (4.0 kB)\n",
            "Collecting llama-index-llms-mistralai<0.3.0,>=0.2.0 (from llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading llama_index_llms_mistralai-0.2.7-py3-none-any.whl.metadata (3.5 kB)\n",
            "Collecting llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0 (from llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading llama_index_postprocessor_cohere_rerank-0.2.1-py3-none-any.whl.metadata (723 bytes)\n",
            "Collecting sentence-transformers>=2.3.0 (from llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading sentence_transformers-3.2.0-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-experimental) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-experimental) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-index-experimental) (2024.2)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.14.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (4.0.3)\n",
            "Requirement already satisfied: torch>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-embeddings-adapter<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.4.1+cu121)\n",
            "Collecting azure-identity<2.0.0,>=1.15.0 (from llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading azure_identity-1.19.0-py3-none-any.whl.metadata (80 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.6/80.6 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: llama-index-llms-openai<0.3.0,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.2.15)\n",
            "Collecting mistralai>=1.0.0 (from llama-index-llms-mistralai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading mistralai-1.1.0-py3-none-any.whl.metadata (23 kB)\n",
            "Collecting cohere<6.0.0,>=5.1.1 (from llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading cohere-5.11.0-py3-none-any.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk>3.8.1->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2024.9.11)\n",
            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.23.4 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.7.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.23.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-index-experimental) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (2024.8.30)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (4.44.2)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.5.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.13.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.24.7)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.1.1)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.0.0)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.23.0)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (3.7.1)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.0.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.14.0)\n",
            "Collecting azure-core>=1.31.0 (from azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading azure_core-1.31.0-py3-none-any.whl.metadata (39 kB)\n",
            "Requirement already satisfied: cryptography>=2.5 in /usr/local/lib/python3.10/dist-packages (from azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (43.0.1)\n",
            "Collecting msal>=1.30.0 (from azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading msal-1.31.0-py3-none-any.whl.metadata (11 kB)\n",
            "Collecting msal-extensions>=1.2.0 (from azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading msal_extensions-1.2.0-py3-none-any.whl.metadata (7.6 kB)\n",
            "Collecting boto3<2.0.0,>=1.34.0 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading boto3-1.35.43-py3-none-any.whl.metadata (6.7 kB)\n",
            "Collecting fastavro<2.0.0,>=1.9.4 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading fastavro-1.9.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.5 kB)\n",
            "Collecting httpx-sse==0.4.0 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading httpx_sse-0.4.0-py3-none-any.whl.metadata (9.0 kB)\n",
            "Collecting parameterized<0.10.0,>=0.9.0 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading parameterized-0.9.0-py2.py3-none-any.whl.metadata (18 kB)\n",
            "Collecting sagemaker<3.0.0,>=2.232.1 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading sagemaker-2.232.2-py3-none-any.whl.metadata (16 kB)\n",
            "Requirement already satisfied: tokenizers<1,>=0.15 in /usr/local/lib/python3.10/dist-packages (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.19.1)\n",
            "Collecting types-requests<3.0.0,>=2.0.0 (from cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading types_requests-2.32.0.20241016-py3-none-any.whl.metadata (1.9 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.16.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (24.1)\n",
            "Requirement already satisfied: openai<2.0.0,>=1.40.0 in /usr/local/lib/python3.10/dist-packages (from llama-index-llms-openai<0.3.0,>=0.2.1->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.52.0)\n",
            "Requirement already satisfied: eval-type-backport<0.3.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from mistralai>=1.0.0->llama-index-llms-mistralai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.2.0)\n",
            "Collecting jsonpath-python<2.0.0,>=1.0.6 (from mistralai>=1.0.0->llama-index-llms-mistralai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading jsonpath_python-1.0.6-py3-none-any.whl.metadata (12 kB)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.0->llama-index-embeddings-adapter<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.13.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.0.0->llama-index-embeddings-adapter<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.1.4)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.4.5)\n",
            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (0.2.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental) (1.2.2)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers>=2.3.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.5.0)\n",
            "Collecting botocore<1.36.0,>=1.35.43 (from boto3<2.0.0,>=1.34.0->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading botocore-1.35.43-py3-none-any.whl.metadata (5.7 kB)\n",
            "Collecting jmespath<2.0.0,>=0.7.1 (from boto3<2.0.0,>=1.34.0->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading jmespath-1.0.1-py3-none-any.whl.metadata (7.6 kB)\n",
            "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3<2.0.0,>=1.34.0->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading s3transfer-0.10.3-py3-none-any.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography>=2.5->azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.17.1)\n",
            "Requirement already satisfied: PyJWT<3,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from PyJWT[crypto]<3,>=1.0.0->msal>=1.30.0->azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.9.0)\n",
            "Collecting portalocker<3,>=1.4 (from msal-extensions>=1.2.0->azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai<0.3.0,>=0.2.1->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.7.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->llama-index-llms-openai<0.3.0,>=0.2.1->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.6.1)\n",
            "Collecting attrs>=17.3.0 (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.12.0,>=0.11.0->llama-index-experimental)\n",
            "  Downloading attrs-23.2.0-py3-none-any.whl.metadata (9.5 kB)\n",
            "Requirement already satisfied: cloudpickle==2.2.1 in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.2.1)\n",
            "Collecting docker (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading docker-7.1.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "Requirement already satisfied: google-pasta in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.2.0)\n",
            "Collecting importlib-metadata<7.0,>=1.4.0 (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading importlib_metadata-6.11.0-py3-none-any.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (4.23.0)\n",
            "Collecting pathos (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading pathos-0.3.3-py3-none-any.whl.metadata (11 kB)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (4.3.6)\n",
            "Requirement already satisfied: protobuf<5.0,>=3.12 in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.20.3)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (5.9.5)\n",
            "Collecting sagemaker-core<2.0.0,>=1.0.0 (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading sagemaker_core-1.0.10-py3-none-any.whl.metadata (4.9 kB)\n",
            "Collecting sagemaker-mlflow (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading sagemaker_mlflow-0.1.0-py3-none-any.whl.metadata (3.3 kB)\n",
            "Collecting schema (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading schema-0.7.7-py2.py3-none-any.whl.metadata (34 kB)\n",
            "Collecting smdebug-rulesconfig==1.0.1 (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl.metadata (943 bytes)\n",
            "Requirement already satisfied: tblib<4,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.0.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.0.0->llama-index-embeddings-adapter<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.0.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=2.0.0->llama-index-embeddings-adapter<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.3.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2.0.0,>=1.15.0->llama-index-llms-azure-openai<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.22)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=1.4.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.20.2)\n",
            "Requirement already satisfied: rich<14.0.0,>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from sagemaker-core<2.0.0,>=1.0.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (13.9.2)\n",
            "Collecting mock<5.0,>4.0 (from sagemaker-core<2.0.0,>=1.0.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading mock-4.0.3-py3-none-any.whl.metadata (2.8 kB)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2024.10.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.35.1)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.20.0)\n",
            "Collecting ppft>=1.7.6.9 (from pathos->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading ppft-1.7.6.9-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting dill>=0.3.9 (from pathos->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading dill-0.3.9-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting pox>=0.3.5 (from pathos->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading pox-0.3.5-py3-none-any.whl.metadata (8.0 kB)\n",
            "Collecting multiprocess>=0.70.17 (from pathos->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading multiprocess-0.70.17-py310-none-any.whl.metadata (7.2 kB)\n",
            "Collecting mlflow>=2.8 (from sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading mlflow-2.17.0-py3-none-any.whl.metadata (29 kB)\n",
            "Collecting mlflow-skinny==2.17.0 (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading mlflow_skinny-2.17.0-py3-none-any.whl.metadata (30 kB)\n",
            "Requirement already satisfied: Flask<4 in /usr/local/lib/python3.10/dist-packages (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.2.5)\n",
            "Collecting alembic!=1.10.0,<2 (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading alembic-1.13.3-py3-none-any.whl.metadata (7.4 kB)\n",
            "Collecting graphene<4 (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading graphene-3.4-py2.py3-none-any.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: markdown<4,>=3.3 in /usr/local/lib/python3.10/dist-packages (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.7)\n",
            "Requirement already satisfied: matplotlib<4 in /usr/local/lib/python3.10/dist-packages (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.7.1)\n",
            "Requirement already satisfied: pyarrow<18,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (16.1.0)\n",
            "Collecting gunicorn<24 (from mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading gunicorn-23.0.0-py3-none-any.whl.metadata (4.4 kB)\n",
            "Requirement already satisfied: cachetools<6,>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (5.5.0)\n",
            "Collecting databricks-sdk<1,>=0.20.0 (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading databricks_sdk-0.35.0-py3-none-any.whl.metadata (38 kB)\n",
            "Collecting gitpython<4,>=3.1.9 (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading GitPython-3.1.43-py3-none-any.whl.metadata (13 kB)\n",
            "Requirement already satisfied: opentelemetry-api<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.16.0)\n",
            "Requirement already satisfied: opentelemetry-sdk<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.16.0)\n",
            "Requirement already satisfied: sqlparse<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.5.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.0.0->sagemaker-core<2.0.0,>=1.0.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.0.0->sagemaker-core<2.0.0,>=1.0.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.18.0)\n",
            "Collecting Mako (from alembic!=1.10.0,<2->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading Mako-1.3.5-py3-none-any.whl.metadata (2.9 kB)\n",
            "Requirement already satisfied: Werkzeug>=2.2.2 in /usr/local/lib/python3.10/dist-packages (from Flask<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.0.4)\n",
            "Requirement already satisfied: itsdangerous>=2.0 in /usr/local/lib/python3.10/dist-packages (from Flask<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.2.0)\n",
            "Collecting graphql-core<3.3,>=3.1 (from graphene<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading graphql_core-3.2.5-py3-none-any.whl.metadata (10 kB)\n",
            "Collecting graphql-relay<3.3,>=3.1 (from graphene<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading graphql_relay-3.2.0-py3-none-any.whl.metadata (12 kB)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.0.0->sagemaker-core<2.0.0,>=1.0.0->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.1.2)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (3.1.4)\n",
            "Requirement already satisfied: google-auth~=2.0 in /usr/local/lib/python3.10/dist-packages (from databricks-sdk<1,>=0.20.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (2.27.0)\n",
            "Collecting gitdb<5,>=4.0.1 (from gitpython<4,>=3.1.9->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading gitdb-4.0.11-py3-none-any.whl.metadata (1.2 kB)\n",
            "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api<3,>=1.9.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (71.0.4)\n",
            "Requirement already satisfied: opentelemetry-semantic-conventions==0.37b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk<3,>=1.9.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.37b0)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython<4,>=3.1.9->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental)\n",
            "  Downloading smmap-5.0.1-py3-none-any.whl.metadata (4.3 kB)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth~=2.0->databricks-sdk<1,>=0.20.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.4.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth~=2.0->databricks-sdk<1,>=0.20.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (4.9)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth~=2.0->databricks-sdk<1,>=0.20.0->mlflow-skinny==2.17.0->mlflow>=2.8->sagemaker-mlflow->sagemaker<3.0.0,>=2.232.1->cohere<6.0.0,>=5.1.1->llama-index-postprocessor-cohere-rerank<0.3.0,>=0.2.0->llama-index-finetuning<0.3.0,>=0.2.0->llama-index-experimental) (0.6.1)\n",
            "Downloading llama_index_experimental-0.4.0-py3-none-any.whl (13 kB)\n",
            "Downloading llama_index_finetuning-0.2.1-py3-none-any.whl (30 kB)\n",
            "Downloading llama_index_embeddings_adapter-0.2.2-py3-none-any.whl (4.5 kB)\n",
            "Downloading llama_index_llms_azure_openai-0.2.2-py3-none-any.whl (6.3 kB)\n",
            "Downloading llama_index_llms_mistralai-0.2.7-py3-none-any.whl (6.8 kB)\n",
            "Downloading llama_index_postprocessor_cohere_rerank-0.2.1-py3-none-any.whl (2.9 kB)\n",
            "Downloading sentence_transformers-3.2.0-py3-none-any.whl (255 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m255.2/255.2 kB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading azure_identity-1.19.0-py3-none-any.whl (187 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m187.6/187.6 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading cohere-5.11.0-py3-none-any.whl (249 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m249.2/249.2 kB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx_sse-0.4.0-py3-none-any.whl (7.8 kB)\n",
            "Downloading mistralai-1.1.0-py3-none-any.whl (229 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m229.7/229.7 kB\u001b[0m \u001b[31m20.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading azure_core-1.31.0-py3-none-any.whl (197 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m197.4/197.4 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading boto3-1.35.43-py3-none-any.whl (139 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fastavro-1.9.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m84.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading jsonpath_python-1.0.6-py3-none-any.whl (7.6 kB)\n",
            "Downloading msal-1.31.0-py3-none-any.whl (113 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.1/113.1 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading msal_extensions-1.2.0-py3-none-any.whl (19 kB)\n",
            "Downloading parameterized-0.9.0-py2.py3-none-any.whl (20 kB)\n",
            "Downloading sagemaker-2.232.2-py3-none-any.whl (1.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m71.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading smdebug_rulesconfig-1.0.1-py2.py3-none-any.whl (20 kB)\n",
            "Downloading attrs-23.2.0-py3-none-any.whl (60 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading types_requests-2.32.0.20241016-py3-none-any.whl (15 kB)\n",
            "Downloading botocore-1.35.43-py3-none-any.whl (12.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.6/12.6 MB\u001b[0m \u001b[31m112.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n",
            "Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
            "Downloading portalocker-2.10.1-py3-none-any.whl (18 kB)\n",
            "Downloading s3transfer-0.10.3-py3-none-any.whl (82 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.6/82.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sagemaker_core-1.0.10-py3-none-any.whl (388 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.4/388.4 kB\u001b[0m \u001b[31m31.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading docker-7.1.0-py3-none-any.whl (147 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.8/147.8 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pathos-0.3.3-py3-none-any.whl (82 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading sagemaker_mlflow-0.1.0-py3-none-any.whl (24 kB)\n",
            "Downloading schema-0.7.7-py2.py3-none-any.whl (18 kB)\n",
            "Downloading dill-0.3.9-py3-none-any.whl (119 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.4/119.4 kB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mlflow-2.17.0-py3-none-any.whl (26.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.7/26.7 MB\u001b[0m \u001b[31m33.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mlflow_skinny-2.17.0-py3-none-any.whl (5.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m87.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Downloading multiprocess-0.70.17-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pox-0.3.5-py3-none-any.whl (29 kB)\n",
            "Downloading ppft-1.7.6.9-py3-none-any.whl (56 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading alembic-1.13.3-py3-none-any.whl (233 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.2/233.2 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading graphene-3.4-py2.py3-none-any.whl (114 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.6/114.6 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gunicorn-23.0.0-py3-none-any.whl (85 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.0/85.0 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading databricks_sdk-0.35.0-py3-none-any.whl (568 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m568.4/568.4 kB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading graphql_core-3.2.5-py3-none-any.whl (203 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m203.2/203.2 kB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading graphql_relay-3.2.0-py3-none-any.whl (16 kB)\n",
            "Downloading Mako-1.3.5-py3-none-any.whl (78 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
            "Installing collected packages: schema, types-requests, smmap, smdebug-rulesconfig, ppft, pox, portalocker, parameterized, mock, Mako, jsonpath-python, jmespath, importlib-metadata, httpx-sse, gunicorn, graphql-core, fastavro, dill, attrs, multiprocess, graphql-relay, gitdb, docker, botocore, azure-core, alembic, s3transfer, pathos, mistralai, graphene, gitpython, databricks-sdk, msal, mlflow-skinny, boto3, sentence-transformers, sagemaker-core, msal-extensions, mlflow, llama-index-llms-mistralai, llama-index-embeddings-adapter, sagemaker-mlflow, azure-identity, sagemaker, llama-index-llms-azure-openai, cohere, llama-index-postprocessor-cohere-rerank, llama-index-finetuning, llama-index-experimental\n",
            "  Attempting uninstall: importlib-metadata\n",
            "    Found existing installation: importlib_metadata 8.5.0\n",
            "    Uninstalling importlib_metadata-8.5.0:\n",
            "      Successfully uninstalled importlib_metadata-8.5.0\n",
            "  Attempting uninstall: attrs\n",
            "    Found existing installation: attrs 24.2.0\n",
            "    Uninstalling attrs-24.2.0:\n",
            "      Successfully uninstalled attrs-24.2.0\n",
            "Successfully installed Mako-1.3.5 alembic-1.13.3 attrs-23.2.0 azure-core-1.31.0 azure-identity-1.19.0 boto3-1.35.43 botocore-1.35.43 cohere-5.11.0 databricks-sdk-0.35.0 dill-0.3.9 docker-7.1.0 fastavro-1.9.7 gitdb-4.0.11 gitpython-3.1.43 graphene-3.4 graphql-core-3.2.5 graphql-relay-3.2.0 gunicorn-23.0.0 httpx-sse-0.4.0 importlib-metadata-6.11.0 jmespath-1.0.1 jsonpath-python-1.0.6 llama-index-embeddings-adapter-0.2.2 llama-index-experimental-0.4.0 llama-index-finetuning-0.2.1 llama-index-llms-azure-openai-0.2.2 llama-index-llms-mistralai-0.2.7 llama-index-postprocessor-cohere-rerank-0.2.1 mistralai-1.1.0 mlflow-2.17.0 mlflow-skinny-2.17.0 mock-4.0.3 msal-1.31.0 msal-extensions-1.2.0 multiprocess-0.70.17 parameterized-0.9.0 pathos-0.3.3 portalocker-2.10.1 pox-0.3.5 ppft-1.7.6.9 s3transfer-0.10.3 sagemaker-2.232.2 sagemaker-core-1.0.10 sagemaker-mlflow-0.1.0 schema-0.7.7 sentence-transformers-3.2.0 smdebug-rulesconfig-1.0.1 smmap-5.0.1 types-requests-2.32.0.20241016\n"
          ]
        }
      ],
      "source": [
        "# aaron new\n",
        "%pip install llama-index-embeddings-openai\n",
        "%pip install llama-index-readers-file pymupdf\n",
        "%pip install llama-index-llms-openai\n",
        "%pip install llama-index-experimental"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "JT3sU8CRGw3l",
        "outputId": "02572fd3-0d92-4837-8fd2-e63533c01fdd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# aaron new\n",
        "\n",
        "import camelot\n",
        "\n",
        "# https://en.wikipedia.org/wiki/The_World%27s_Billionaires\n",
        "from llama_index.core import VectorStoreIndex\n",
        "from llama_index.experimental.query_engine import PandasQueryEngine\n",
        "from llama_index.core.schema import IndexNode\n",
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "from llama_index.readers.file import PyMuPDFReader\n",
        "from typing import List"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398
        },
        "id": "BqJQDrZqzAEp",
        "outputId": "1af1ec33-6cd2-487e-8d03-fc1a80d88e6a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "ImportError",
          "evalue": "cannot import name 'Document' from 'llama_index' (unknown location)",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-38-fe8d841b3896>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcamelot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mllama_index\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDocument\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSummaryIndex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# https://en.wikipedia.org/wiki/The_World%27s_Billionaires\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mllama_index\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVectorStoreIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mServiceContext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLLMPredictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mImportError\u001b[0m: cannot import name 'Document' from 'llama_index' (unknown location)",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "import camelot\n",
        "from llama_index import Document, SummaryIndex\n",
        "\n",
        "# https://en.wikipedia.org/wiki/The_World%27s_Billionaires\n",
        "from llama_index import VectorStoreIndex, ServiceContext, LLMPredictor\n",
        "from llama_index.query_engine import PandasQueryEngine, RetrieverQueryEngine\n",
        "from llama_index.retrievers import RecursiveRetriever\n",
        "from llama_index.schema import IndexNode\n",
        "from llama_index.llms import OpenAI\n",
        "\n",
        "from llama_hub.file.pymu_pdf.base import PyMuPDFReader\n",
        "from pathlib import Path\n",
        "from typing import List"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ok7qgGuyzRMx"
      },
      "source": [
        "### Parse out Table, build Pandas Query Engine"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 575
        },
        "id": "-FyhtIy43ZWQ",
        "outputId": "5de06145-f95d-4f0a-9459-ce30f9938b30"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2023-11-08 08:39:04--  https://www.dropbox.com/scl/fi/waoz9bo9yiemnhnqvu0cc/billionaires_page.pdf?rlkey=4i08msa7zr1lpnuq2y1vs2xgw&dl=1\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.67.18, 2620:100:6025:18::a27d:4512\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.67.18|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com/cd/0/inline/CHILPsnmWsPf2A36gkigVOpcEGOEPC5ozf159kcAuew7RElvDPzsnCSDtWgUAhofhJWPHM7BUp-qrES7oGQnSWKUQBGFH126pEgeN72xQsYTBjw4h5Yq56h2KBJ5w3Rrx20/file?dl=1# [following]\n",
            "--2023-11-08 08:39:04--  https://uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com/cd/0/inline/CHILPsnmWsPf2A36gkigVOpcEGOEPC5ozf159kcAuew7RElvDPzsnCSDtWgUAhofhJWPHM7BUp-qrES7oGQnSWKUQBGFH126pEgeN72xQsYTBjw4h5Yq56h2KBJ5w3Rrx20/file?dl=1\n",
            "Resolving uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com (uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com)... 162.125.71.15, 2620:100:6021:15::a27d:410f\n",
            "Connecting to uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com (uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com)|162.125.71.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /cd/0/inline2/CHKjbWpWIGqoqrztPc1KS0mhb7GEzH3F_Tl7PnbSNZaXwZRHts8t_rGMnHT5EDLnPRocjJu3VvNdEVfw1GZcL7OKJbIpaAHpTEpHoIKQmZIG5jXnrVQe75OG3_z0veVD81LBYoDrKD8vYpHqk6257ZOZuq7W05m_VOkARB219b31V_CI0HFBmNRicWw83b4PDrBYj0n9ecB4TMXts8HEjjG7eQAK03z2qi8dOe9RVud5XAENDle-l3LXHRhiqClbP-buWdRKtSIS4PXHOpEDazrt1n7er4JXaR-GNQvnEzaqpCHRGh83P9h9Q0anQZ7xt8AajYFUM0P-iqKCYYyMZBA9korX3RKYNHYTdJyEmninNw/file?dl=1 [following]\n",
            "--2023-11-08 08:39:05--  https://uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com/cd/0/inline2/CHKjbWpWIGqoqrztPc1KS0mhb7GEzH3F_Tl7PnbSNZaXwZRHts8t_rGMnHT5EDLnPRocjJu3VvNdEVfw1GZcL7OKJbIpaAHpTEpHoIKQmZIG5jXnrVQe75OG3_z0veVD81LBYoDrKD8vYpHqk6257ZOZuq7W05m_VOkARB219b31V_CI0HFBmNRicWw83b4PDrBYj0n9ecB4TMXts8HEjjG7eQAK03z2qi8dOe9RVud5XAENDle-l3LXHRhiqClbP-buWdRKtSIS4PXHOpEDazrt1n7er4JXaR-GNQvnEzaqpCHRGh83P9h9Q0anQZ7xt8AajYFUM0P-iqKCYYyMZBA9korX3RKYNHYTdJyEmninNw/file?dl=1\n",
            "Reusing existing connection to uc19051c67f1e7f1353ff651529f.dl.dropboxusercontent.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3063791 (2.9M) [application/binary]\n",
            "Saving to: ‘billionaires_page.pdf’\n",
            "\n",
            "billionaires_page.p 100%[===================>]   2.92M  2.16MB/s    in 1.4s    \n",
            "\n",
            "2023-11-08 08:39:07 (2.16 MB/s) - ‘billionaires_page.pdf’ saved [3063791/3063791]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget \"https://www.dropbox.com/scl/fi/waoz9bo9yiemnhnqvu0cc/billionaires_page.pdf?rlkey=4i08msa7zr1lpnuq2y1vs2xgw&dl=1\" -O billionaires_page.pdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "M6QVhLgYH7BN",
        "outputId": "41c76524-8c62-4309-8c73-833eec0967a4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cp: cannot stat '/Users/aaron/ws/# 2024/70 菱光社 奇美LLM開發維運/單張/2023表1.pdf': No such file or directory\n"
          ]
        }
      ],
      "source": [
        "!cp /Users/aaron/ws/\"# 2024\"/\"70 菱光社 奇美LLM開發維運/單張/2023表1.pdf\" ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "926BvznbIS_m",
        "outputId": "90bba56d-33f4-4b17-ba2c-fed19fea87a1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cp: cannot stat '/Users/aaron/ws/2023表1.pdf': No such file or directory\n"
          ]
        }
      ],
      "source": [
        "!cp /Users/aaron/ws/2023表1.pdf ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "bk7Aj3x1zC0P",
        "outputId": "ad17f7a3-6d02-45d6-c215-d6bd99e8a8c5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# initialize PDF reader\n",
        "reader = PyMuPDFReader()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "63mCpEZZL-Ai",
        "outputId": "d2131bf5-a0cc-4f9d-946d-e77656c50e2d"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "id": "qs6m889qMRlW",
        "outputId": "1b26c9dd-7c4d-4c64-95fd-692d2ebb9c03"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 2023年度合併財務報表.pdf   2023表4.pdf\n",
            " 2023文表1-5.pdf\t    ah_openai_embed_test01.ipynb\n",
            " 2023表1.pdf\t\t   'Copy of Chat with Data Notebook (Embedded Tables, extended)'\n",
            " 2023表2.pdf\t\t    ngrok_test.ipynb\n",
            " 2023表3.pdf\t\t    openai_test02.ipynb\n"
          ]
        }
      ],
      "source": [
        "!ls /content/drive/MyDrive/'Colab Notebooks'/\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "9dDN5OUBMkHw",
        "outputId": "723b478c-d880-424b-ce32-7247c8d4438e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "'/content/drive/MyDrive/Colab Notebooks/2023表1.pdf'\n"
          ]
        }
      ],
      "source": [
        "!ls /content/drive/MyDrive/'Colab Notebooks'/2023表1.pdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "A48MFvCHzELc",
        "outputId": "eb57dca2-e7ad-4113-e31f-95465a652e2a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#file_path = \"billionaires_page.pdf\"\n",
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表1.pdf'\n",
        "docs = reader.load(file_path=file_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "qUFUJFMjzFaR",
        "outputId": "6aed0655-b13d-44ee-9cce-46f77b64a745"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# use camelot to parse tables\n",
        "def get_tables(path: str, pages: List[int]):\n",
        "    table_dfs = []\n",
        "    for page in pages:\n",
        "        table_list = camelot.read_pdf(path, pages=str(page))\n",
        "        table_df = table_list[0].df\n",
        "        table_df = (\n",
        "            table_df.rename(columns=table_df.iloc[0])\n",
        "            .drop(table_df.index[0])\n",
        "            .reset_index(drop=True)\n",
        "        )\n",
        "        table_dfs.append(table_df)\n",
        "    return table_dfs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "id": "prSCnWjWzGvN",
        "outputId": "e9344f9e-f2ff-457a-fae3-5f17684c0dc3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "IndexError",
          "evalue": "list index out of range",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-57-4289e3ab4e89>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtable_dfs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_tables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpages\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-50-950b84e83774>\u001b[0m in \u001b[0;36mget_tables\u001b[0;34m(path, pages)\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mpage\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpages\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mtable_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcamelot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_pdf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpages\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0mtable_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtable_list\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m         table_df = (\n\u001b[1;32m      8\u001b[0m             \u001b[0mtable_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtable_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/camelot/core.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 689\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tables\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: list index out of range"
          ]
        }
      ],
      "source": [
        "table_dfs = get_tables(file_path, pages=[1, 1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ZeKbNIEaPmSF",
        "outputId": "1191a225-7ab4-48f8-e5f5-087182e3644e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of tables found: 0\n"
          ]
        }
      ],
      "source": [
        "# aaron test:\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\")\n",
        "print(f\"Number of tables found: {len(table_list)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "id": "zLDXK5MCQEbg",
        "outputId": "88318268-6be9-46c1-ac2c-307720ba84e9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " 2023年度合併財務報表.pdf   2023表4.pdf\n",
            " 2023文表1-5.pdf\t    ah_openai_embed_test01.ipynb\n",
            " 2023表1.pdf\t\t   'Copy of Chat with Data Notebook (Embedded Tables, extended)'\n",
            " 2023表2.pdf\t\t    ngrok_test.ipynb\n",
            " 2023表3.pdf\t\t    openai_test02.ipynb\n"
          ]
        }
      ],
      "source": [
        "!ls '/content/drive/MyDrive/Colab Notebooks/'\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "xRGX9w6vP66H",
        "outputId": "215e5309-da70-45ed-d887-309672360ca7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of tables found: 0\n"
          ]
        }
      ],
      "source": [
        "# file_path = '/content/drive/MyDrive/Colab Notebooks/2023表1.pdf'\n",
        "# file_path = '/content/drive/MyDrive/Colab Notebooks/2023表2.pdf'\n",
        "# file_path = '/content/drive/MyDrive/Colab Notebooks/2023表3.pdf'\n",
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表4.pdf'\n",
        "# file_path = '/content/drive/MyDrive/Colab Notebooks/2023文表1-5.pdf'\n",
        "# file_path = '/content/drive/MyDrive/Colab Notebooks/2023年度合併財務報表.pdf'\n",
        "# aaron test:\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\")\n",
        "print(f\"Number of tables found: {len(table_list)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Ljf3azbfRGs1",
        "outputId": "cf305399-791d-4c58-9f7d-e2c08e16954c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表4.pdf'\n",
        "\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\", flavor=\"stream\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "djYMh1OeRRtP",
        "outputId": "3ace3d5b-87ad-434f-93de-d96f465f5c6e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<Table shape=(39, 4)>"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "table_list[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        },
        "id": "OoMm39G5Reqm",
        "outputId": "05957a14-b033-4332-92be-44e1dc3e64bb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "TypeError",
          "evalue": "cannot unpack non-iterable Table object",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-69-28f5a21af573>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtable_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtable_list\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: cannot unpack non-iterable Table object"
          ]
        }
      ],
      "source": [
        "for i, j in table_list:\n",
        "  print(table_list[i,j])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 741
        },
        "id": "2JQ6F9GYSJxF",
        "outputId": "f06f7703-6d33-4083-ab36-9a410cd7e400"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                       0  1            2            3\n",
            "0                                            單位：新台幣千元\n",
            "1                                  112年度        111年度\n",
            "2             營業活動之現金流量：                             \n",
            "3              本期稅前淨(損)利  $  (1,112,021)    5,558,149\n",
            "4                  調整項目：                             \n",
            "5                 收益費損項目                             \n",
            "6                   折舊費用       5,022,749    4,541,542\n",
            "7                   攤銷費用         280,470      217,831\n",
            "8         預期信用減損(迴轉利益)損失        (32,244)          400\n",
            "9                   利息費用         611,465      495,913\n",
            "10                  利息收入       (770,955)    (794,577)\n",
            "11                  股利收入       (226,618)    (181,318)\n",
            "12                員工酬勞成本         381,133      838,895\n",
            "13  採用權益法認列之關聯企業及合資利益之份額        (68,820)     (38,396)\n",
            "14  處分及報廢不動產、廠房及設備損失(利益)          11,466     (36,519)\n",
            "15             非金融資產減損損失         200,911            -\n",
            "16                    其他         132,420      (4,746)\n",
            "17              收益費損項目合計       5,541,977    5,039,025\n",
            "18     與營業活動相關之資產／負債變動數：                             \n",
            "19       與營業活動相關之資產之淨變動：                             \n",
            "20    強制透過損益按公允價值衡量之金融資產       (229,955)    5,227,828\n",
            "21               應收票據及帳款       (161,263)    2,699,493\n",
            "22                    存貨         279,265    4,659,944\n",
            "23                其他流動資產       (145,211)    1,946,548\n",
            "24      與營業活動相關之資產之淨變動合計       (257,164)   14,533,813\n",
            "25       與營業活動相關之負債之淨變動：                             \n",
            "26               應付票據及帳款     (1,587,158)  (1,390,089)\n",
            "27                 其他應付款     (1,220,978)  (2,962,390)\n",
            "28                其他流動負債       1,217,654    (838,140)\n",
            "29               淨確定福利負債        (67,633)     (65,307)\n",
            "30      與營業活動相關之負債之淨變動合計     (1,658,115)  (5,255,926)\n",
            "31   與營業活動相關之資產及負債之淨變動合計     (1,915,279)    9,277,887\n",
            "32                調整項目合計       3,626,698   14,316,912\n",
            "33             營運產生之現金流入       2,514,677   19,875,061\n",
            "34                 收取之利息         820,145      739,024\n",
            "35                 收取之股利         243,965      203,414\n",
            "36                 支付之利息       (624,419)    (428,185)\n",
            "37                支付之所得稅     (1,112,732)  (6,206,376)\n",
            "38            營業活動之淨現金流入       1,841,636   14,182,938\n"
          ]
        }
      ],
      "source": [
        "#aaron\n",
        "# Get the table as a pandas DataFrame\n",
        "table_df = table_list[0].df\n",
        "\n",
        "# Print the DataFrame\n",
        "print(table_df)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "D6nH4pS4UL7l"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "id": "Tf-uOUpGQ2XK",
        "outputId": "214bf327-77d3-4bee-d7a8-5df7be160e0b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "NameError",
          "evalue": "name 'table_dfs' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-65-2b8d6c9767eb>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtable_dfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'table_dfs' is not defined"
          ]
        }
      ],
      "source": [
        "table_dfs[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "H3fTdo8nT2iI",
        "outputId": "4928f9ee-7402-4340-ce53-ff1ccb09e958"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<Table shape=(31, 14)>"
            ]
          },
          "execution_count": 72,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表3.pdf'\n",
        "\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\", flavor=\"stream\")\n",
        "table_list[0]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "hqaKwiWIURs7",
        "outputId": "9355f527-0421-4ae4-80a6-4164e4c409da"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                       0  1           2          3           4            5   \\\n",
            "0                                   股　　本                                保留盈餘   \n",
            "1                                                                              \n",
            "2                                    普通股                    法定盈          特別盈   \n",
            "3                                    股　本       資本公積         餘公積          餘公積   \n",
            "4            民國一一一年一月一日餘額  $  18,197,753  1,726,136  14,842,179    2,682,297   \n",
            "5                    本期淨利              -          -           -            -   \n",
            "6                本期其他綜合損益              -          -           -            -   \n",
            "7                本期綜合損益總額              -          -           -            -   \n",
            "8                盈餘指撥及分配：                                                      \n",
            "9                提列法定盈餘公積              -          -   2,050,808            -   \n",
            "10                普通股現金股利              -          -           -            -   \n",
            "11               特別盈餘公積迴轉              -          -           -  (2,682,297)   \n",
            "12                  庫藏股買回              -          -           -            -   \n",
            "13  實際取得或處分子公司股權價格與帳面價值差額              -       (11)           -            -   \n",
            "14            對子公司所有權權益變動              -    (2,841)           -            -   \n",
            "15               股份基礎給付交易              -     84,864           -            -   \n",
            "16                非控制權益增減              -          -           -            -   \n",
            "17        民國一一一年十二月三十一日餘額     18,197,753  1,808,148  16,892,987            -   \n",
            "18                本期淨利(損)              -          -           -            -   \n",
            "19               本期其他綜合損益              -          -           -            -   \n",
            "20               本期綜合損益總額              -          -           -            -   \n",
            "21               盈餘指撥及分配：                                                      \n",
            "22               提列法定盈餘公積              -          -     324,286            -   \n",
            "23               提列特別盈餘公積              -          -           -      528,261   \n",
            "24                普通股現金股利              -          -           -            -   \n",
            "25                  庫藏股買回              -          -           -            -   \n",
            "26  實際取得或處分子公司股權價格與帳面價值差額              -          1           -            -   \n",
            "27            對子公司所有權權益變動              -        217           -            -   \n",
            "28               股份基礎給付交易       (10,505)     12,002           -            -   \n",
            "29                非控制權益增減              -          -           -            -   \n",
            "30        民國一一二年十二月三十一日餘額  $  18,187,248  1,820,368  17,217,273      528,261   \n",
            "\n",
            "              6            7            8            9          10  \\\n",
            "0                       國外營運機        合損益按公                           \n",
            "1                       構財務報表        允價值衡量                           \n",
            "2            未分配        換算之兌換        之金融資產        員 工 未              \n",
            "3            盈　餘        差　　　額        未實現損益         賺得酬勞       庫藏股票   \n",
            "4     58,644,270  (1,091,739)    1,406,581  (1,343,531)  (150,772)   \n",
            "5      2,809,866            -            -            -          -   \n",
            "6        443,685      563,407  (1,406,510)            -          -   \n",
            "7      3,253,551      563,407  (1,406,510)            -          -   \n",
            "8                                                                    \n",
            "9    (2,050,808)            -            -            -          -   \n",
            "10  (10,709,351)            -            -            -          -   \n",
            "11     2,682,297            -            -            -          -   \n",
            "12             -            -            -            -   (62,768)   \n",
            "13             -            -            -            -          -   \n",
            "14      (10,693)            -            -            -          -   \n",
            "15             -            -            -      753,628    134,587   \n",
            "16             -            -            -            -          -   \n",
            "17    51,809,266    (528,332)           71    (589,903)   (78,953)   \n",
            "18        93,756            -            -            -          -   \n",
            "19      (62,359)    (570,534)    (101,790)            -          -   \n",
            "20        31,397    (570,534)    (101,790)            -          -   \n",
            "21                                                                   \n",
            "22     (324,286)            -            -            -          -   \n",
            "23     (528,261)            -            -            -          -   \n",
            "24   (5,385,025)            -            -            -          -   \n",
            "25             -            -            -            -   (41,280)   \n",
            "26             -            -            -            -          -   \n",
            "27             -            -            -            -          -   \n",
            "28             -            -            -      379,063    107,240   \n",
            "29             -            -            -            -          -   \n",
            "30    45,603,091  (1,098,866)    (101,719)    (210,840)   (12,993)   \n",
            "\n",
            "              11           12            13  \n",
            "0                                            \n",
            "1           歸屬於母                             \n",
            "2           公司業主          非控制                \n",
            "3           權益總計          權　益          權益總額  \n",
            "4     94,913,174   12,598,887   107,512,061  \n",
            "5      2,809,866      202,101     3,011,967  \n",
            "6      (399,418)      200,173     (199,245)  \n",
            "7      2,410,448      402,274     2,812,722  \n",
            "8                                            \n",
            "9              -            -             -  \n",
            "10  (10,709,351)            -  (10,709,351)  \n",
            "11             -            -             -  \n",
            "12      (62,768)            -      (62,768)  \n",
            "13          (11)            -          (11)  \n",
            "14      (13,534)            -      (13,534)  \n",
            "15       973,079            -       973,079  \n",
            "16             -  (1,964,466)   (1,964,466)  \n",
            "17    87,511,037   11,036,695    98,547,732  \n",
            "18        93,756    (759,825)     (666,069)  \n",
            "19     (734,683)    (155,930)     (890,613)  \n",
            "20     (640,927)    (915,755)   (1,556,682)  \n",
            "21                                           \n",
            "22             -            -             -  \n",
            "23             -            -             -  \n",
            "24   (5,385,025)            -   (5,385,025)  \n",
            "25      (41,280)            -      (41,280)  \n",
            "26             1            -             1  \n",
            "27           217            -           217  \n",
            "28       487,800            -       487,800  \n",
            "29             -     (20,848)      (20,848)  \n",
            "30    81,931,823   10,100,092    92,031,915  \n"
          ]
        }
      ],
      "source": [
        "#aaron\n",
        "# Get the table as a pandas DataFrame\n",
        "table_df = table_list[0].df\n",
        "\n",
        "# Print the DataFrame\n",
        "print(table_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jbHLFJDXXC7t"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "IsJ1kvgYXC1c",
        "outputId": "3f74be3f-a058-474f-b1d9-1b058718cb98"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<Table shape=(31, 14)>"
            ]
          },
          "execution_count": 78,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表1.pdf'\n",
        "\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\", flavor=\"stream\")\n",
        "table_list[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Tjkf06VrU3-I",
        "outputId": "90ecbdf7-7f94-48ce-e11c-37c8dd7e9379"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "      0                                   1  2              3    4   \\\n",
            "0                                                    112.12.31        \n",
            "1                                       資　　產              金　　額    %   \n",
            "2                                      流動資產：                          \n",
            "3   1100                   現金及約當現金(附註四及六(一))  $     17,656,357   14   \n",
            "4   1110       透過損益按公允價值衡量之金融資產－流動(附註四及六(二))        14,446,307   11   \n",
            "5   1170               應收票據及帳款淨額(附註四、六(三)及七)         9,731,631    8   \n",
            "6   1200                   其他應收款(附註四、六(四)及七)           185,364    -   \n",
            "7   130X                          存貨(附註六(五))        17,318,635   13   \n",
            "8   1476                    其他金融資產－流動(附註四及八)            12,300    -   \n",
            "9   1479                   其他流動資產－其他(附註六(十))         1,911,214    1   \n",
            "10                                                  61,261,808   47   \n",
            "11                                    非流動資產：                          \n",
            "12  1517  透過其他綜合損益按公允價值衡量之金融資產－非流動(附註四及六(二))           811,937    1   \n",
            "13  1550                  採用權益法之投資(附註四及六(六))           579,991    -   \n",
            "14  1600                 不動產、廠房及設備(附註四及六(七))        63,753,118   49   \n",
            "15  1755                     使用權資產(附註四及六(八))         2,203,334    2   \n",
            "16  1780                      無形資產(附註四及六(九))           489,180    -   \n",
            "17  1840                  遞延所得稅資產(附註四及六(十七))           798,049    1   \n",
            "18  1990                其他非流動資產－其他(附註六(十)及八)           411,339    -   \n",
            "19                                                  69,046,948   53   \n",
            "20                                                                    \n",
            "21                                                                    \n",
            "22                                                                    \n",
            "23                                                                    \n",
            "24                                                                    \n",
            "25                                                                    \n",
            "26                                                                    \n",
            "27                                                                    \n",
            "28                                                                    \n",
            "29                                                                    \n",
            "30                                      資產總計     $ 130,308,756  100   \n",
            "\n",
            "             5    6     7                              8  9              10  \\\n",
            "0     111.12.31                                                   112.12.31   \n",
            "1          金　　額    %                                負債及權益              金　　額   \n",
            "2                                                   流動負債：                     \n",
            "3    38,005,094   26  2100                  短期借款(附註六(十一))  $      1,680,269   \n",
            "4    14,216,352   10  2110                應付短期票券(附註六(十二))            99,863   \n",
            "5     9,664,022    7  2170                   應付票據及帳款(附註七)         4,079,431   \n",
            "6       187,607    -  2200                     其他應付款(附註七)         4,494,568   \n",
            "7    17,715,519   12  2230             本期所得稅負債(附註四及六(十七))            65,909   \n",
            "8        10,319    -  2280             租賃負債－流動(附註四及六(十五))           170,511   \n",
            "9     2,118,110    2  2320  一年或一營業週期內到期長期負債(附註六(十三)及(十四))         4,671,077   \n",
            "10   81,917,023   57  2399        其他流動負債－其他(附註六(二)、(九)及七)         3,463,875   \n",
            "11                                                               18,725,503   \n",
            "12      920,231    1                               非流動負債：                     \n",
            "13      523,838    -  2530               應付公司債(附註四及六(十四))         3,100,000   \n",
            "14   58,185,754   40  2540                  長期借款(附註六(十三))         9,797,787   \n",
            "15    1,466,604    1  2570             遞延所得稅負債(附註四及六(十七))         5,527,711   \n",
            "16      291,566    -  2580            租賃負債－非流動(附註四及六(十五))           800,915   \n",
            "17      784,631    1  2640         淨確定福利負債－非流動(附註四及六(十六))           142,130   \n",
            "18      359,713    -  2670                其他非流動負債－其他(附註七)           182,795   \n",
            "19   62,532,337   43                                             19,551,338   \n",
            "20                                                   負債總計        38,276,841   \n",
            "21                                            歸屬母公司業主之權益：                     \n",
            "22                    3100                    股本(附註六(十八))        18,187,248   \n",
            "23                    3200                  資本公積(附註六(十八))         1,820,368   \n",
            "24                    3300                  保留盈餘(附註六(十八))        63,348,625   \n",
            "25                    3400                  其他權益(附註六(十八))       (1,411,425)   \n",
            "26                    3500                  庫藏股票(附註六(十八))          (12,993)   \n",
            "27                                          歸屬於母公司業主之權益合計        81,931,823   \n",
            "28                    36XX                 非控制權益(附註六(十八))        10,100,092   \n",
            "29                                                   權益總計        92,031,915   \n",
            "30  144,449,360  100                              負債及權益總計     $ 130,308,756   \n",
            "\n",
            "     11           12   13  \n",
            "0          111.12.31       \n",
            "1     %         金　　額    %  \n",
            "2                          \n",
            "3     1    4,140,213    3  \n",
            "4     -            -    -  \n",
            "5     3    5,723,445    4  \n",
            "6     3    6,423,045    4  \n",
            "7     -    1,179,080    1  \n",
            "8     -      136,440    -  \n",
            "9     4    8,527,360    6  \n",
            "10    3    2,246,235    2  \n",
            "11   14   28,375,818   20  \n",
            "12                         \n",
            "13    2    1,000,000    1  \n",
            "14    8   10,189,851    7  \n",
            "15    4    5,990,071    4  \n",
            "16    1       45,987    -  \n",
            "17    -      131,481    -  \n",
            "18    -      168,420    -  \n",
            "19   15   17,525,810   12  \n",
            "20   29   45,901,628   32  \n",
            "21                         \n",
            "22   14   18,197,753   13  \n",
            "23    1    1,808,148    1  \n",
            "24   49   68,702,253   47  \n",
            "25  (1)  (1,118,164)  (1)  \n",
            "26    -     (78,953)    -  \n",
            "27   63   87,511,037   60  \n",
            "28    8   11,036,695    8  \n",
            "29   71   98,547,732   68  \n",
            "30  100  144,449,360  100  \n"
          ]
        }
      ],
      "source": [
        "#aaron\n",
        "# Get the table as a pandas DataFrame\n",
        "table_df = table_list[0].df\n",
        "\n",
        "# Print the DataFrame\n",
        "print(table_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MwUjQE3MU34m"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "f6cbBXd3UJDd",
        "outputId": "e090af2c-ee64-497a-c467-34d5dd45d26f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "<Table shape=(44, 7)>"
            ]
          },
          "execution_count": 75,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "file_path = '/content/drive/MyDrive/Colab Notebooks/2023表2.pdf'\n",
        "\n",
        "table_list = camelot.read_pdf(file_path, pages=\"1\", flavor=\"stream\")\n",
        "table_list[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 832
        },
        "id": "xpZ1qvYkVQoe",
        "outputId": "877c8d84-8fc5-4ba8-ac11-8246137d201f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "       0                             1  2            3    4            5    6\n",
            "0                                                112年度             111年度     \n",
            "1                                                  金　額    ％          金　額    ％\n",
            "2   4000            營業收入(附註四、六(二十一)及七)  $  124,821,623  100  153,538,492  100\n",
            "3   5000                  營業成本(附註六(五))     118,324,275   95  138,078,028   90\n",
            "4                                 營業毛利       6,497,348    5   15,460,464   10\n",
            "5                    營業費用：(附註六(二十二)及七)                                       \n",
            "6   6100                          推銷費用       5,045,834    4    6,942,057    5\n",
            "7   6200                          管理費用       2,145,332    2    2,231,793    1\n",
            "8   6300                        研究發展費用       1,043,840    1    1,185,725    1\n",
            "9   6450                  預期信用減損(利益)損失        (32,244)    -          400    -\n",
            "10                                           8,202,762    7   10,359,975    7\n",
            "11                             營業淨(損)利     (1,705,414)  (2)    5,100,489    3\n",
            "12                           營業外收入及支出：                                       \n",
            "13  7100                利息收入(附註六(二十三))         770,955    1      794,577    1\n",
            "14  7010              其他收入(附註六(二十三)及七)         276,442    -      235,204    -\n",
            "15  7020           其他利益及損失(附註六(二十三)及七)          88,641    -    (114,604)    -\n",
            "16  7050          財務成本(附註六(七)、(二十三)及七)       (611,465)    -    (495,913)    -\n",
            "17  7060  採用權益法認列之關聯企業及合資損益之份額(附註六(六))          68,820    -       38,396    -\n",
            "18                                             593,393    1      457,660    1\n",
            "19  7900                       稅前淨(損)利     (1,112,021)  (1)    5,558,149    4\n",
            "20  7950          減：所得稅(利益)費用(附註六(十七))       (445,952)    -    2,546,182    2\n",
            "21  8200                       本期淨(損)利       (666,069)  (1)    3,011,967    2\n",
            "22                             其他綜合損益：                                       \n",
            "23  8310                    不重分類至損益之項目                                       \n",
            "24  8311                   確定福利計畫之再衡量數        (77,933)    -      553,900    -\n",
            "25  8316  透過其他綜合損益按公允價值衡量之權益工具投資未實現評價損       (101,594)    -  (1,405,960)  (1)\n",
            "26                                   益                                       \n",
            "27  8349              減：與不重分類之項目相關之所得稅          15,455    -    (109,315)    -\n",
            "28                                           (164,072)    -    (961,375)  (1)\n",
            "29  8360                 後續可能重分類至損益之項目                                       \n",
            "30  8361             國外營運機構財務報表換算之兌換差額       (721,220)  (1)      789,608    1\n",
            "31  8370     採用權益法認列之關聯企業及合資之其他綜合損益之份額         (5,321)    -     (27,478)    -\n",
            "32  8399             減：與可能重分類之項目相關之所得稅               -    -            -    -\n",
            "33                                           (726,541)  (1)      762,130    1\n",
            "34                      本期其他綜合損益(稅後淨額)       (890,613)  (1)    (199,245)    -\n",
            "35  8500                      本期綜合損益總額  $  (1,556,682)  (2)    2,812,722    2\n",
            "36                         本期淨(損)利歸屬於：                                       \n",
            "37  8610                         母公司業主  $       93,756    -    2,809,866    2\n",
            "38  8620                         非控制權益       (759,825)  (1)      202,101    -\n",
            "39                                      $    (666,069)  (1)    3,011,967    2\n",
            "40                          綜合損益總額歸屬於：                                       \n",
            "41  8710                         母公司業主  $    (640,927)  (1)    2,410,448    2\n",
            "42  8720                         非控制權益       (915,755)  (1)      402,274    -\n",
            "43                                      $  (1,556,682)  (2)    2,812,722    2\n"
          ]
        }
      ],
      "source": [
        "#aaron\n",
        "# Get the table as a pandas DataFrame\n",
        "table_df = table_list[0].df\n",
        "\n",
        "# Print the DataFrame\n",
        "print(table_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        },
        "id": "R7_MxDU_zIEx",
        "outputId": "82b3eed4-0af8-44cc-efb9-075df411dea5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "NameError",
          "evalue": "name 'table_dfs' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-76-caba260fd1e5>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# shows list of top billionaires in 2023\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtable_dfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'table_dfs' is not defined"
          ]
        }
      ],
      "source": [
        "# shows list of top billionaires in 2023\n",
        "table_dfs[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 833
        },
        "id": "wnvCdO78zJ3v",
        "outputId": "1778ef06-3e3b-4275-fc84-ba672adb0aba"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-0f8f935e-0e12-45ea-adf9-168ac16e281a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Year</th>\n",
              "      <th>Number of billionaires</th>\n",
              "      <th>Group's combined net worth</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2023[2]</td>\n",
              "      <td>2,640</td>\n",
              "      <td>$12.2 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2022[6]</td>\n",
              "      <td>2,668</td>\n",
              "      <td>$12.7 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2021[11]</td>\n",
              "      <td>2,755</td>\n",
              "      <td>$13.1 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020</td>\n",
              "      <td>2,095</td>\n",
              "      <td>$8.0 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019</td>\n",
              "      <td>2,153</td>\n",
              "      <td>$8.7 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2018</td>\n",
              "      <td>2,208</td>\n",
              "      <td>$9.1 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2017</td>\n",
              "      <td>2,043</td>\n",
              "      <td>$7.7 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2016</td>\n",
              "      <td>1,810</td>\n",
              "      <td>$6.5 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2015[18]</td>\n",
              "      <td>1,826</td>\n",
              "      <td>$7.1 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>2014[67]</td>\n",
              "      <td>1,645</td>\n",
              "      <td>$6.4 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>2013[68]</td>\n",
              "      <td>1,426</td>\n",
              "      <td>$5.4 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>2012</td>\n",
              "      <td>1,226</td>\n",
              "      <td>$4.6 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>2011</td>\n",
              "      <td>1,210</td>\n",
              "      <td>$4.5 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2010</td>\n",
              "      <td>1,011</td>\n",
              "      <td>$3.6 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>2009</td>\n",
              "      <td>793</td>\n",
              "      <td>$2.4 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>2008</td>\n",
              "      <td>1,125</td>\n",
              "      <td>$4.4 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>2007</td>\n",
              "      <td>946</td>\n",
              "      <td>$3.5 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>2006</td>\n",
              "      <td>793</td>\n",
              "      <td>$2.6 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>2005</td>\n",
              "      <td>691</td>\n",
              "      <td>$2.2 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>2004</td>\n",
              "      <td>587</td>\n",
              "      <td>$1.9 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2003</td>\n",
              "      <td>476</td>\n",
              "      <td>$1.4 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2002</td>\n",
              "      <td>497</td>\n",
              "      <td>$1.5 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2001</td>\n",
              "      <td>538</td>\n",
              "      <td>$1.8 trillion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>2000</td>\n",
              "      <td>470</td>\n",
              "      <td>$898 billion</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>Sources: Forbes.[18][67][66][68]</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0f8f935e-0e12-45ea-adf9-168ac16e281a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0f8f935e-0e12-45ea-adf9-168ac16e281a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0f8f935e-0e12-45ea-adf9-168ac16e281a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-4cd543a9-963e-4d30-a56d-db408fa96c8f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4cd543a9-963e-4d30-a56d-db408fa96c8f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-4cd543a9-963e-4d30-a56d-db408fa96c8f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                Year Number of billionaires  \\\n",
              "0                            2023[2]                  2,640   \n",
              "1                            2022[6]                  2,668   \n",
              "2                           2021[11]                  2,755   \n",
              "3                               2020                  2,095   \n",
              "4                               2019                  2,153   \n",
              "5                               2018                  2,208   \n",
              "6                               2017                  2,043   \n",
              "7                               2016                  1,810   \n",
              "8                           2015[18]                  1,826   \n",
              "9                           2014[67]                  1,645   \n",
              "10                          2013[68]                  1,426   \n",
              "11                              2012                  1,226   \n",
              "12                              2011                  1,210   \n",
              "13                              2010                  1,011   \n",
              "14                              2009                    793   \n",
              "15                              2008                  1,125   \n",
              "16                              2007                    946   \n",
              "17                              2006                    793   \n",
              "18                              2005                    691   \n",
              "19                              2004                    587   \n",
              "20                              2003                    476   \n",
              "21                              2002                    497   \n",
              "22                              2001                    538   \n",
              "23                              2000                    470   \n",
              "24  Sources: Forbes.[18][67][66][68]                          \n",
              "\n",
              "   Group's combined net worth  \n",
              "0              $12.2 trillion  \n",
              "1              $12.7 trillion  \n",
              "2              $13.1 trillion  \n",
              "3               $8.0 trillion  \n",
              "4               $8.7 trillion  \n",
              "5               $9.1 trillion  \n",
              "6               $7.7 trillion  \n",
              "7               $6.5 trillion  \n",
              "8               $7.1 trillion  \n",
              "9               $6.4 trillion  \n",
              "10              $5.4 trillion  \n",
              "11              $4.6 trillion  \n",
              "12              $4.5 trillion  \n",
              "13              $3.6 trillion  \n",
              "14              $2.4 trillion  \n",
              "15              $4.4 trillion  \n",
              "16              $3.5 trillion  \n",
              "17              $2.6 trillion  \n",
              "18              $2.2 trillion  \n",
              "19              $1.9 trillion  \n",
              "20              $1.4 trillion  \n",
              "21              $1.5 trillion  \n",
              "22              $1.8 trillion  \n",
              "23               $898 billion  \n",
              "24                             "
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# shows list of top billionaires\n",
        "table_dfs[1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "Q-NdLsc9zLZg",
        "outputId": "c1f39034-4673-4b14-974c-ea48a209a0a5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /tmp/llama_index...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ],
      "source": [
        "# define query engines over these tables\n",
        "df_query_engines = [PandasQueryEngine(table_df) for table_df in table_dfs]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "tmksOoRuzMxe",
        "outputId": "e6b9fcfc-14ee-4654-a4dc-3d3bf4200c43"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "$180 billion\n"
          ]
        }
      ],
      "source": [
        "response = df_query_engines[0].query(\n",
        "    \"What's the net worth of the second richest billionaire in 2023?\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "iYeIkl0wzN4H",
        "outputId": "7cb30624-3ed4-4da7-8ba9-d98001d5ef3f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "793\n"
          ]
        }
      ],
      "source": [
        "response = df_query_engines[1].query(\"How many billionaires were there in 2009?\")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPhTPOALzTf8"
      },
      "source": [
        "### Build Recursive Retriever\n",
        "\n",
        "We define a top-level vector index that does top-k lookup over a set of Nodes. We define two special nodes (`IndexNode` objects) linking to each of these tables.\n",
        "\n",
        "We define a `RecursiveRetriever` object to recursively retrieve/query nodes. We then put this in our `RetrieverQueryEngine` along with a `ResponseSynthesizer` to synthesize a response.\n",
        "\n",
        "We pass in mappings from id to retriever and id to query engine. We then pass in a root id representing the retriever we query first."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Q16ooq0vzenz",
        "outputId": "a03d0573-8f32-4476-db72-c9967c165dc9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "llm = OpenAI(temperature=0, model=\"gpt-4\")\n",
        "\n",
        "service_context = ServiceContext.from_defaults(\n",
        "    llm=llm,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "HzEleylizfl9",
        "outputId": "2de31aaa-b3b2-4b84-96c9-b66d2117c4dc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "doc_nodes = service_context.node_parser.get_nodes_from_documents(docs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "Ix5wBtHTzg8o",
        "outputId": "4b240741-885a-4f60-b029-dd696ed119a2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# define index nodes\n",
        "summaries = [\n",
        "    \"This node provides information about the world's richest billionaires in 2023\",\n",
        "    \"This node provides information on the number of billionaires and their combined net worth from 2000 to 2023.\",\n",
        "]\n",
        "\n",
        "df_nodes = [\n",
        "    IndexNode(text=summary, index_id=f\"pandas{idx}\")\n",
        "    for idx, summary in enumerate(summaries)\n",
        "]\n",
        "\n",
        "df_id_query_engine_mapping = {\n",
        "    f\"pandas{idx}\": df_query_engine\n",
        "    for idx, df_query_engine in enumerate(df_query_engines)\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "I_8SoJwEzh6C",
        "outputId": "3eec8820-88af-4745-92ff-c0e3095f59e0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# construct top-level vector index + query engine\n",
        "vector_index = VectorStoreIndex(doc_nodes + df_nodes)\n",
        "vector_retriever = vector_index.as_retriever(similarity_top_k=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "_IlcUcCQzpdG",
        "outputId": "f476e6a3-b8c7-4201-c3f4-fbdfc48e78fb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from llama_index.retrievers import RecursiveRetriever\n",
        "from llama_index.query_engine import RetrieverQueryEngine\n",
        "from llama_index.response_synthesizers import get_response_synthesizer\n",
        "\n",
        "recursive_retriever = RecursiveRetriever(\n",
        "    \"vector\",\n",
        "    retriever_dict={\"vector\": vector_retriever},\n",
        "    query_engine_dict=df_id_query_engine_mapping,\n",
        "    verbose=True,\n",
        ")\n",
        "\n",
        "response_synthesizer = get_response_synthesizer(\n",
        "    # service_context=service_context,\n",
        "    response_mode=\"compact\"\n",
        ")\n",
        "\n",
        "query_engine = RetrieverQueryEngine.from_args(\n",
        "    recursive_retriever, response_synthesizer=response_synthesizer\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ol-2hQ9hzqp_"
      },
      "source": [
        "### Define Baseline Retriever\n",
        "\n",
        "We also define a baseline retriever that does top-k lookup over the raw document."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "KxMk4zehipBf",
        "outputId": "4cdc5cb3-30b6-4144-abb4-f740cface9e8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# baseline vector index (that doesn't include the extra df nodes).\n",
        "# used to benchmark\n",
        "vector_index0 = VectorStoreIndex(doc_nodes)\n",
        "vector_query_engine0 = vector_index0.as_query_engine()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YRsB_28_isCA"
      },
      "source": [
        "### Compare Results\n",
        "\n",
        "We compare results between the recursive retriever vs. baseline retriever."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "Ymguy0KGztxd",
        "outputId": "9fcb5ae8-ea65-45c3-df73-faecdca71f8e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;3;34mRetrieving with query id None: How many billionaires were there in 2009?\n",
            "\u001b[0m\u001b[1;3;38;5;200mRetrieved node with id, entering: pandas1\n",
            "\u001b[0m\u001b[1;3;34mRetrieving with query id pandas1: How many billionaires were there in 2009?\n",
            "\u001b[0m\u001b[1;3;32mGot response: 793\n",
            "\u001b[0m793\n"
          ]
        }
      ],
      "source": [
        "response = query_engine.query(\n",
        "    \"How many billionaires were there in 2009?\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "TIM0KPcYz2t2",
        "outputId": "cf41d562-7ae3-43cc-a7e3-48b4b4ec43b6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Query: How many billionaires were there in 2009?\\nResponse: 793'"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "response.source_nodes[0].node.get_content()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "Ylgg7AYYzxLd",
        "outputId": "7c682903-18ef-4bc0-f1ac-3f0e2632f108"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Based on the context information, it is not possible to determine the exact number of billionaires in 2009.\n"
          ]
        }
      ],
      "source": [
        "response = vector_query_engine0.query(\n",
        "    \"How many billionaires were there in 2009?\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "MCLtfVdNz5Op",
        "outputId": "6be1aa74-a7b0-42de-8fc5-f7ce82d36600"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "7/1/23, 11:31 PM\n",
            "The World's Billionaires - Wikipedia\n",
            "https://en.wikipedia.org/wiki/The_World%27s_Billionaires\n",
            "6/33\n",
            "In the 32nd annual Forbes list of the world's billionaires, the aggregate wealth of the top 20 richest\n",
            "people on Earth amounted to about 13 percent of all billionaires' fortunes combined.[15] A record of\n",
            "2,208 billionaires were in the ranking and the total wealth was $9.1 trillion, up 18% since 2017. For\n",
            "the first time, Jeff Bezos was listed as the top billionaire due to Amazon's rising stock price that\n",
            "resulted in one person's biggest one-year gain in wealth ($35 billion) since Forbes started tracking in\n",
            "1987.[15] The U.S. had the most billionaires in the world, with 585, while China was catching up with\n",
            "476 when including Hong Kong, Macau and Taiwan; it had 372 when excluding those three places.[15]\n",
            "Forbes excluded Al-Walid bin Talal and all other Saudi billionaires due to the absence of accurate\n",
            "wealth estimations as a result of the 2017–19 Saudi Arabian purge.[16]\n",
            "No.\n",
            "Name\n",
            "Net worth (USD)\n",
            "Age\n",
            "Nationality\n",
            "Source(s) of wealth\n",
            "1 \n",
            "Jeff Bezos\n",
            "$112 billion \n",
            "54\n",
            " United States\n",
            "Amazon\n",
            "2 \n",
            "Bill Gates\n",
            "$90 billion \n",
            "62\n",
            " United States\n",
            "Microsoft\n",
            "3 \n",
            "Warren Buffett\n",
            "$84 billion \n",
            "87\n",
            " United States\n",
            "Berkshire Hathaway\n",
            "4 \n",
            "Bernard Arnault\n",
            "$72 billion \n",
            "69\n",
            " France\n",
            "LVMH\n",
            "5 \n",
            "Mark Zuckerberg\n",
            "$71 billion \n",
            "33\n",
            " United States\n",
            "Facebook, Inc.\n",
            "6 \n",
            "Amancio Ortega\n",
            "$70 billion \n",
            "81\n",
            " Spain\n",
            "Inditex, Zara\n",
            "7 \n",
            "Carlos Slim\n",
            "$67.1 billion \n",
            "78\n",
            " Mexico\n",
            "América Móvil, Grupo Carso\n",
            "8 \n",
            "Charles Koch\n",
            "$60 billion \n",
            "82\n",
            " United States\n",
            "Koch Industries\n",
            "8 \n",
            "David Koch\n",
            "$60 billion \n",
            "77\n",
            " United States\n",
            "Koch Industries\n",
            "10 \n",
            "Larry Ellison\n",
            "$58.5 billion \n",
            "73\n",
            " United States\n",
            "Oracle Corporation\n",
            "On the 30th anniversary of the Forbes list of the world's billionaires, for the fourth year in a row, Bill\n",
            "Gates was named the richest man in the world.[5] In 2017, there was a record of 2,043 people on the\n",
            "list, which is the first time over 2,000 people were listed. This included 195 newcomers of whom 76\n",
            "were from China and 25 from the U.S.; there were 56 people under 40 and it had a record of 227\n",
            "women.[17] The number of billionaires increased 13% to 2,043 from 1,810 in 2016; this was the biggest\n",
            "change in over 30 years of tracking billionaires globally.[5] Added together, the total net worth for\n",
            "2017's billionaires was US$7.67 trillion, up from US$7.1 trillion in 2015.[5][18] This was the first time\n",
            "after 12 years that Carlos Slim was not within the top five. The U.S. had the most billionaires in the\n",
            "world, with a record of 565. China had 319 (not including Hong Kong, Taiwan or Macau), Germany\n",
            "had 114, and India had the fourth most with 101; India reached over 100 billionaires for its first time.\n",
            "2017\n"
          ]
        }
      ],
      "source": [
        "print(response.source_nodes[1].node.get_content())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "id": "kQZhiiIJjB_s",
        "outputId": "9f0197c3-8331-434e-ea00-18a402f061cb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;3;34mRetrieving with query id None: What is the average age of top 5 billionaires in 2023? Make sure age is a float.\n",
            "\u001b[0m\u001b[1;3;38;5;200mRetrieved node with id, entering: pandas0\n",
            "\u001b[0m\u001b[1;3;34mRetrieving with query id pandas0: What is the average age of top 5 billionaires in 2023? Make sure age is a float.\n",
            "\u001b[0m\u001b[1;3;32mGot response: 70.8\n",
            "\u001b[0mThe average age of the top 5 billionaires in 2023 is 70.8.\n"
          ]
        }
      ],
      "source": [
        "response = query_engine.query(\n",
        "    \"What is the average age of top 5 billionaires in 2023? Make sure age is a float.\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "gbWsicqqjXYi",
        "outputId": "c017b4fc-baa4-42b9-93ef-f4facbc46822"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The average age of the top 5 billionaires in 2023 is 72.2.\n"
          ]
        }
      ],
      "source": [
        "response = vector_query_engine0.query(\n",
        "    \"What is the average age of top 5 billionaires in 2023? Make sure age is a float.\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PVEtBgX4kiaj"
      },
      "source": [
        "Of course, just like the baseline vector query engine, the recursive retriever can answer semantic queries over the article as well."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "cpe2JqBfkf7K",
        "outputId": "64556143-ae02-4699-997b-17807ddb6151"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[1;3;34mRetrieving with query id None: How is wealth accounted for in recipients if the billionaire is deceased?\n",
            "\u001b[0m\u001b[1;3;38;5;200mRetrieving text node: 7/1/23, 11:31 PM\n",
            "The World's Billionaires - Wikipedia\n",
            "https://en.wikipedia.org/wiki/The_World%27s_Billionaires\n",
            "2/33\n",
            "stock are priced to market on a date roughly a month before publication. Privately held companies are\n",
            "priced by the prevailing price-to-sales or price-to-earnings ratios. Known debt is subtracted from\n",
            "assets to get a final estimate of an individual's estimated worth in United States dollars. Since stock\n",
            "prices fluctuate rapidly, an individual's true wealth and ranking at the time of publication may vary\n",
            "from their situation when the list was compiled.[7]\n",
            "When a living individual has dispersed his or her wealth to immediate family members it is included\n",
            "under a single listing (as a single \"family fortune\") provided that individual (the grantor) is still living.\n",
            "However, if a deceased billionaire's fortune has been dispersed, it will not appear as a single listing,\n",
            "and each recipient will only appear if his or her own total net worth is over a $Billion (his or her net\n",
            "worth will not be combined with family members')[7] Royal families and dictators that have their\n",
            "wealth contingent on a position are always excluded from these lists.[9]\n",
            "The rankings are published annually in March, so the net worths listed are snapshots taken at that\n",
            "time. These lists only show the top 10 wealthiest billionaires for each year.\n",
            "Icon\n",
            "Description\n",
            "Has not changed from the previous ranking.\n",
            "Has increased from the previous ranking.\n",
            "Has decreased from the previous ranking.\n",
            "In the 37th annual Forbes list of the world's billionaires, the list included 2,640 billionaires with a\n",
            "total net wealth of $12.2 trillion, down 28 members and $500 billion from 2022. Nearly half the list is\n",
            "poorer than the previous year, including Elon Musk, who fell from No. 1 to No. 2.[2] The list also\n",
            "marks for the first time a French citizen was in the top position as well as a non-American for the first\n",
            "time since 2013 when the Mexican Carlos Slim Helu was the world's richest person. The list, like in\n",
            "2022, counted 15 under 30 billionaires with the richest of them being Red Bull heir Mark Mateschitz\n",
            "with a net worth of $34.7 billion. The youngest of the lot were Clemente Del Vecchio, heir to the\n",
            "Luxottica fortune shared with his six siblings and stepmother and Kim Jung-yang, whose fortune lies\n",
            "in Japanese-South Korean gaming giant Nexon, both under-20s.[10]\n",
            "Annual rankings\n",
            "Legend\n",
            "2023\n",
            "\u001b[0mIf a deceased billionaire's fortune has been dispersed, it will not appear as a single listing. Instead, each recipient will only appear if their own total net worth is over a billion dollars. In other words, the net worth of each individual recipient will not be combined with that of other family members.\n"
          ]
        }
      ],
      "source": [
        "response = query_engine.query(\n",
        "    \"How is wealth accounted for in recipients if the billionaire is deceased?\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "IPZJRbJnk_s4",
        "outputId": "e1a917b1-abe1-40f8-b55e-523c5f153b7b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "If a deceased billionaire's fortune has been dispersed, it will not appear as a single listing. Instead, each recipient will only appear if their own total net worth is over a billion dollars. In other words, the net worth of each individual recipient will not be combined with that of other family members.\n"
          ]
        }
      ],
      "source": [
        "response = vector_query_engine0.query(\n",
        "    \"How is wealth accounted for in recipients if the billionaire is deceased?\"\n",
        ")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3ZaPyxG_4j2"
      },
      "source": [
        "## Extended Use Case: Parsing Tesla 10Qs\n",
        "\n",
        "Here we deal with an even messier document format - parsing tables within a 10K.\n",
        "\n",
        "We use the Unstructured library to help us extract tables.\n",
        "\n",
        "The tables aren't perfectly formatted, but work well enough."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "oJT0MpiCCqiK",
        "outputId": "1e3985da-df4f-489a-b628-8400be5ed1d4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting unstructured\n",
            "  Downloading unstructured-0.10.29-py3-none-any.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (from unstructured) (5.2.0)\n",
            "Collecting filetype (from unstructured)\n",
            "  Downloading filetype-1.2.0-py2.py3-none-any.whl (19 kB)\n",
            "Collecting python-magic (from unstructured)\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.9.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from unstructured) (2.31.0)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.11.2)\n",
            "Collecting emoji (from unstructured)\n",
            "  Downloading emoji-2.8.0-py2.py3-none-any.whl (358 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m358.9/358.9 kB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: dataclasses-json in /usr/local/lib/python3.10/dist-packages (from unstructured) (0.5.14)\n",
            "Collecting python-iso639 (from unstructured)\n",
            "  Downloading python_iso639-2023.6.15-py3-none-any.whl (275 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m275.1/275.1 kB\u001b[0m \u001b[31m19.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langdetect (from unstructured)\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unstructured) (1.23.5)\n",
            "Collecting rapidfuzz (from unstructured)\n",
            "  Downloading rapidfuzz-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m44.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting backoff (from unstructured)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from unstructured) (4.5.0)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->unstructured) (2.5)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json->unstructured) (0.9.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->unstructured) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (1.3.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->unstructured) (4.66.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (1.26.18)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->unstructured) (2023.7.22)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.10/dist-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->unstructured) (23.2)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json->unstructured) (1.0.0)\n",
            "Building wheels for collected packages: langdetect\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993224 sha256=0bb73d4428e2d33042026c0aa9289324682e0cd7a4cf8d9edecc7f6a7bc1a81f\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n",
            "Successfully built langdetect\n",
            "Installing collected packages: filetype, rapidfuzz, python-magic, python-iso639, langdetect, emoji, backoff, unstructured\n",
            "Successfully installed backoff-2.2.1 emoji-2.8.0 filetype-1.2.0 langdetect-1.0.9 python-iso639-2023.6.15 python-magic-0.4.27 rapidfuzz-3.5.2 unstructured-0.10.29\n"
          ]
        }
      ],
      "source": [
        "!pip install unstructured"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-rgNa4raBP91"
      },
      "source": [
        "### Extract Elements"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "_lIcAz6sAEnI",
        "outputId": "c7827bed-2b4c-488d-8057-22427f3a99d4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from pydantic import BaseModel\n",
        "from unstructured.partition.html import partition_html\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "wf5G_ne_AG-Z",
        "outputId": "e2c32758-06f0-4722-ac51-3352509de6a1"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "--2023-11-08 15:51:33--  https://www.dropbox.com/scl/fi/mlaymdy1ni1ovyeykhhuk/tesla_2021_10k.htm?rlkey=qf9k4zn0ejrbm716j0gg7r802&dl=1\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.65.18, 2620:100:601b:18::a27d:812\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.65.18|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com/cd/0/inline/CHLjbU75m2bMyWOrogDeJxZxDTWRHX3aJdMvB30xWHi54AHyO71ak9byoVYTwIWB7wOons_Wnr_w5fqpNv-HwOW2IkWj5tg2_87-PbYhMwV2iF7xl7uD17ut9ulTnPH01ZY/file?dl=1# [following]\n",
            "--2023-11-08 15:51:34--  https://uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com/cd/0/inline/CHLjbU75m2bMyWOrogDeJxZxDTWRHX3aJdMvB30xWHi54AHyO71ak9byoVYTwIWB7wOons_Wnr_w5fqpNv-HwOW2IkWj5tg2_87-PbYhMwV2iF7xl7uD17ut9ulTnPH01ZY/file?dl=1\n",
            "Resolving uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com (uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com)... 162.125.65.15, 2620:100:6022:15::a27d:420f\n",
            "Connecting to uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com (uc8c2f9df597ecdae58e0a717b26.dl.dropboxusercontent.com)|162.125.65.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 7898923 (7.5M) [application/binary]\n",
            "Saving to: ‘tesla_2021_10k.htm’\n",
            "\n",
            "tesla_2021_10k.htm  100%[===================>]   7.53M  34.9MB/s    in 0.2s    \n",
            "\n",
            "2023-11-08 15:51:34 (34.9 MB/s) - ‘tesla_2021_10k.htm’ saved [7898923/7898923]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget \"https://www.dropbox.com/scl/fi/mlaymdy1ni1ovyeykhhuk/tesla_2021_10k.htm?rlkey=qf9k4zn0ejrbm716j0gg7r802&dl=1\" -O tesla_2021_10k.htm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "QhkoN2ayAH7F",
        "outputId": "1ff1f8d8-4a43-46d5-bc1c-6ea5c060f6b3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from llama_index.readers.file.flat_reader import FlatReader\n",
        "from pathlib import Path\n",
        "\n",
        "reader = FlatReader()\n",
        "docs_2021 = reader.load_data(Path(\"tesla_2021_10k.htm\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "omfHCg8IAJ6c",
        "outputId": "ffbc66b5-6396-4634-af7c-d4af3b8d59dd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from llama_index.node_parser import (\n",
        "    UnstructuredElementNodeParser,\n",
        ")\n",
        "\n",
        "node_parser = UnstructuredElementNodeParser()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "id": "B43MNRgSAK4F",
        "outputId": "5c4e4c1e-0580-410c-afa7-2120bc4bb1c0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <style>\n",
              "    pre {\n",
              "        white-space: pre-wrap;\n",
              "    }\n",
              "  </style>\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "100%|██████████| 105/105 [21:36<00:00, 12.35s/it]\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import pickle\n",
        "\n",
        "if not os.path.exists(\"2021_nodes.pkl\"):\n",
        "    raw_nodes_2021 = node_parser.get_nodes_from_documents(docs_2021)\n",
        "    pickle.dump(raw_nodes_2021, open(\"2021_nodes.pkl\", \"wb\"))\n",
        "else:\n",
        "    raw_nodes_2021 = pickle.load(open(\"2021_nodes.pkl\", \"rb\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LQclEz8TAnsc"
      },
      "outputs": [],
      "source": [
        "base_nodes_2021, node_mappings_2021 = node_parser.get_base_nodes_and_mappings(\n",
        "    raw_nodes_2021\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cAh5Q9MrAo7x"
      },
      "outputs": [],
      "source": [
        "example_index_node = [b for b in base_nodes_2021 if isinstance(b, IndexNode)][\n",
        "    20\n",
        "]\n",
        "\n",
        "# Index Node\n",
        "print(\n",
        "    f\"\\n--------\\n{example_index_node.get_content(metadata_mode='all')}\\n--------\\n\"\n",
        ")\n",
        "# Index Node ID\n",
        "print(f\"\\n--------\\nIndex ID: {example_index_node.index_id}\\n--------\\n\")\n",
        "# Referenceed Table\n",
        "print(\n",
        "    f\"\\n--------\\n{node_mappings_2021[example_index_node.index_id].get_content()}\\n--------\\n\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jiEXTOJJArVZ"
      },
      "source": [
        "### Build Recursive Retriever"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HoqKWlF2Asrk"
      },
      "outputs": [],
      "source": [
        "from llama_index.retrievers import RecursiveRetriever\n",
        "from llama_index.query_engine import RetrieverQueryEngine\n",
        "from llama_index import VectorStoreIndex, ServiceContext\n",
        "from llama_index.llms import OpenAI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5WwTWCedAuib"
      },
      "outputs": [],
      "source": [
        "llm = OpenAI(model=\"gpt-4-1106-preview\")\n",
        "service_context = ServiceContext.from_defaults(llm=llm)\n",
        "\n",
        "# construct top-level vector index + query engine\n",
        "vector_index = VectorStoreIndex(base_nodes_2021, service_context=service_context)\n",
        "vector_retriever = vector_index.as_retriever(similarity_top_k=1)\n",
        "vector_query_engine = vector_index.as_query_engine(similarity_top_k=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_vUeCatwAvZ1"
      },
      "outputs": [],
      "source": [
        "from llama_index.retrievers import RecursiveRetriever\n",
        "\n",
        "recursive_retriever = RecursiveRetriever(\n",
        "    \"vector\",\n",
        "    retriever_dict={\"vector\": vector_retriever},\n",
        "    node_dict=node_mappings_2021,\n",
        "    verbose=True,\n",
        ")\n",
        "query_engine = RetrieverQueryEngine.from_args(recursive_retriever)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qurWIG1JAwjK"
      },
      "source": [
        "### Run Some Queries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eFQF6aXdAxTD"
      },
      "outputs": [],
      "source": [
        "response = query_engine.query(\"What was the revenue in 2020?\")\n",
        "print(str(response))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9L0XyEDmAyY9"
      },
      "outputs": [],
      "source": [
        "# compare against the baseline retriever\n",
        "response = vector_query_engine.query(\"What was the revenue in 2020?\")\n",
        "print(str(response))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}