diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 98a47e1e51..ebce2cd4e3 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -444,3 +444,34 @@ jobs: with: file: ./coverage.xml flags: unittest + + LlamaIndexAgent: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ["3.11"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for LlamaIndexConverableAgent + run: | + pip install -e . + pip install llama-index + pip install llama-index-llms-openai + - name: Coverage + run: | + pytest test/agentchat/contrib/test_llamaindex_conversable_agent.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/autogen/agentchat/contrib/llamaindex_conversable_agent.py b/autogen/agentchat/contrib/llamaindex_conversable_agent.py new file mode 100644 index 0000000000..f7a9c3e615 --- /dev/null +++ b/autogen/agentchat/contrib/llamaindex_conversable_agent.py @@ -0,0 +1,109 @@ +from typing import Dict, List, Optional, Tuple, Union + +from autogen import OpenAIWrapper +from autogen.agentchat import Agent, ConversableAgent +from autogen.agentchat.contrib.vectordb.utils import get_logger + +logger = get_logger(__name__) + +try: + from llama_index.core.agent.runner.base import AgentRunner + from llama_index.core.chat_engine.types import AgentChatResponse + from llama_index_client import ChatMessage +except ImportError as e: + logger.fatal("Failed to import llama-index. Try running 'pip install llama-index'") + raise e + + +class LLamaIndexConversableAgent(ConversableAgent): + + def __init__( + self, + name: str, + llama_index_agent: AgentRunner, + description: Optional[str] = None, + **kwargs, + ): + """ + Args: + name (str): agent name. + llama_index_agent (AgentRunner): llama index agent. + Please override this attribute if you want to reprogram the agent. + description (str): a short description of the agent. This description is used by other agents + (e.g. the GroupChatManager) to decide when to call upon this agent. + **kwargs (dict): Please refer to other kwargs in + [ConversableAgent](../conversable_agent#__init__). + """ + + if llama_index_agent is None: + raise ValueError("llama_index_agent must be provided") + + if description is None or description.isspace(): + raise ValueError("description must be provided") + + super().__init__( + name, + description=description, + **kwargs, + ) + + self._llama_index_agent = llama_index_agent + + # Override the `generate_oai_reply` + self.replace_reply_func(ConversableAgent.generate_oai_reply, LLamaIndexConversableAgent._generate_oai_reply) + + self.replace_reply_func(ConversableAgent.a_generate_oai_reply, LLamaIndexConversableAgent._a_generate_oai_reply) + + def _generate_oai_reply( + self, + messages: Optional[List[Dict]] = None, + sender: Optional[Agent] = None, + config: Optional[OpenAIWrapper] = None, + ) -> Tuple[bool, Union[str, Dict, None]]: + """Generate a reply using autogen.oai.""" + user_message, history = self._extract_message_and_history(messages=messages, sender=sender) + + chatResponse: AgentChatResponse = self._llama_index_agent.chat(message=user_message, chat_history=history) + + extracted_response = chatResponse.response + + return (True, extracted_response) + + async def _a_generate_oai_reply( + self, + messages: Optional[List[Dict]] = None, + sender: Optional[Agent] = None, + config: Optional[OpenAIWrapper] = None, + ) -> Tuple[bool, Union[str, Dict, None]]: + """Generate a reply using autogen.oai.""" + user_message, history = self._extract_message_and_history(messages=messages, sender=sender) + + chatResponse: AgentChatResponse = await self._llama_index_agent.achat( + message=user_message, chat_history=history + ) + + extracted_response = chatResponse.response + + return (True, extracted_response) + + def _extract_message_and_history( + self, messages: Optional[List[Dict]] = None, sender: Optional[Agent] = None + ) -> Tuple[str, List[ChatMessage]]: + """Extract the message and history from the messages.""" + if not messages: + messages = self._oai_messages[sender] + + if not messages: + return "", [] + + message = messages[-1].get("content", "") + + history = messages[:-1] + history_messages: List[ChatMessage] = [] + for history_message in history: + content = history_message.get("content", "") + role = history_message.get("role", "user") + if role: + if role == "user" or role == "assistant": + history_messages.append(ChatMessage(content=content, role=role, additional_kwargs={})) + return message, history_messages diff --git a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb new file mode 100644 index 0000000000..ddff0a0109 --- /dev/null +++ b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb @@ -0,0 +1,225 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "9a71fa36", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "# Groupchat with Llamaindex agents\n", + "\n", + "[Llamaindex agents](https://docs.llamaindex.ai/en/stable/optimizing/agentic_strategies/agentic_strategies/) have the ability to use planning strategies to answer user questions. They can be integrated in Autogen in easy ways\n", + "\n", + "## Requirements" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c528cd6d", + "metadata": {}, + "outputs": [], + "source": [ + "! pip install pyautogen\n", + "! pip install llama-index\n", + "! pip install llama-index-tools-wikipedia\n", + "! pip install llama-index-readers-wikipedia\n", + "! pip install wikipedia" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5ebd2397", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "source": [ + "## Set your API Endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "dca301a4", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import autogen\n", + "\n", + "config_list = [{\"model\": \"gpt-3.5-turbo-0125\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" + ] + }, + { + "cell_type": "markdown", + "id": "76c11ea8", + "metadata": {}, + "source": [ + "## Set Llamaindex" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2d3d298e", + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core import Settings\n", + "from llama_index.core.agent import ReActAgent\n", + "from llama_index.embeddings.openai import OpenAIEmbedding\n", + "from llama_index.llms.openai import OpenAI\n", + "from llama_index.tools.wikipedia import WikipediaToolSpec\n", + "\n", + "llm = OpenAI(\n", + " model=\"gpt-3.5-turbo-0125\",\n", + " temperature=0.0,\n", + " api_key=os.environ.get(\"OPENAPI_API_KEY\", \"\"),\n", + ")\n", + "\n", + "embed_model = OpenAIEmbedding(\n", + " model=\"text-embedding-ada-002\",\n", + " temperature=0.0,\n", + " api_key=os.environ.get(\"OPENAPI_API_KEY\", \"\"),\n", + ")\n", + "\n", + "Settings.llm = llm\n", + "Settings.embed_model = embed_model\n", + "\n", + "# create a react agent to use wikipedia tool\n", + "wiki_spec = WikipediaToolSpec()\n", + "# Get the search wikipedia tool\n", + "wikipedia_tool = wiki_spec.to_tool_list()[1]\n", + "\n", + "location_specialist = ReActAgent.from_tools(tools=[wikipedia_tool], llm=llm, max_iterations=10, verbose=True)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "2b9526e7", + "metadata": {}, + "source": [ + "## Create agents\n", + "\n", + "In this example, we will create a Llamaindex agent to answer questions fecting data from wikipedia and a user proxy agent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a10c9fe-1fbc-40c6-b655-5d2256864ce8", + "metadata": {}, + "outputs": [], + "source": [ + "from llamaindex_conversable_agent import LLamaIndexConversableAgent\n", + "\n", + "llm_config = {\n", + " \"temperature\": 0,\n", + " \"config_list\": config_list,\n", + "}\n", + "\n", + "trip_assistant = LLamaIndexConversableAgent(\n", + " \"trip_specialist\",\n", + " llama_index_agent=location_specialist,\n", + " system_message=\"You help customers finding more about places they would like to visit. You can use external resources to provide more details as you engage with the customer.\",\n", + " description=\"This agents helps customers discover locations to visit, things to do, and other details about a location. It can use external resources to provide more details. This agent helps in finding attractions, history and all that there si to know about a place\",\n", + ")\n", + "\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"Admin\",\n", + " human_input_mode=\"ALWAYS\",\n", + " code_execution_config=False,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "966c96a4-cc8a-4400-b8db-a21b7142e33c", + "metadata": {}, + "source": [ + "Next, let's set up our group chat." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "354b4a8f-7a96-455b-9f17-cbc19d880462", + "metadata": {}, + "outputs": [], + "source": [ + "groupchat = autogen.GroupChat(\n", + " agents=[trip_assistant, user_proxy],\n", + " messages=[],\n", + " max_round=500,\n", + " speaker_selection_method=\"round_robin\",\n", + " enable_clear_history=True,\n", + ")\n", + "manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5518947", + "metadata": {}, + "outputs": [], + "source": [ + "chat_result = user_proxy.initiate_chat(\n", + " manager,\n", + " message=\"\"\"\n", + "What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.\n", + "\"\"\",\n", + ")" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Integrate llamaindex agents with Autogen.", + "tags": [ + "react", + "llama index", + "software engineering" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/agentchat/contrib/test_llamaindex_conversable_agent.py b/test/agentchat/contrib/test_llamaindex_conversable_agent.py new file mode 100644 index 0000000000..ec31e1f6a3 --- /dev/null +++ b/test/agentchat/contrib/test_llamaindex_conversable_agent.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 -m pytest + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +import pytest +from conftest import MOCK_OPEN_AI_API_KEY + +from autogen import GroupChat, GroupChatManager +from autogen.agentchat.contrib.llamaindex_conversable_agent import LLamaIndexConversableAgent +from autogen.agentchat.conversable_agent import ConversableAgent + +sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from conftest import reason, skip_openai + +skip_reasons = [reason] +try: + from llama_index.core.agent import ReActAgent + from llama_index.core.chat_engine.types import AgentChatResponse + from llama_index.llms.openai import OpenAI + + skip_for_dependencies = False + skip_reason = "" +except ImportError as e: + skip_for_dependencies = True + skip_reason = f"dependency not installed: {e.msg}" + pass + + +openaiKey = MOCK_OPEN_AI_API_KEY + + +@pytest.mark.skipif(skip_for_dependencies, reason=skip_reason) +@patch("llama_index.core.agent.ReActAgent.chat") +def test_group_chat_with_llama_index_conversable_agent(chat_mock: MagicMock) -> None: + """ + Tests the group chat functionality with two MultimodalConversable Agents. + Verifies that the chat is correctly limited by the max_round parameter. + Each agent is set to describe an image in a unique style, but the chat should not exceed the specified max_rounds. + """ + llm = OpenAI( + model="gpt-4", + temperature=0.0, + api_key=openaiKey, + ) + + chat_mock.return_value = AgentChatResponse( + response="Visit ghibli studio in Tokyo, Japan. It is a must-visit place for fans of Hayao Miyazaki and his movies like Spirited Away." + ) + + location_specialist = ReActAgent.from_tools(llm=llm, max_iterations=5) + + # create an autogen agent using the react agent + trip_assistant = LLamaIndexConversableAgent( + "trip_specialist", + llama_index_agent=location_specialist, + system_message="You help customers finding more about places they would like to visit. You can use external resources to provide more details as you engage with the customer.", + description="This agents helps customers discover locations to visit, things to do, and other details about a location. It can use external resources to provide more details. This agent helps in finding attractions, history and all that there si to know about a place", + ) + + llm_config = False + max_round = 5 + + user_proxy = ConversableAgent( + "customer", + max_consecutive_auto_reply=10, + human_input_mode="NEVER", + llm_config=False, + default_auto_reply="Thank you. TERMINATE", + ) + + group_chat = GroupChat( + agents=[user_proxy, trip_assistant], + messages=[], + max_round=100, + send_introductions=False, + speaker_selection_method="round_robin", + ) + + group_chat_manager = GroupChatManager( + groupchat=group_chat, + llm_config=llm_config, + is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0, + ) + + # Initiating the group chat and observing the number of rounds + user_proxy.initiate_chat( + group_chat_manager, + message="What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.", + ) + + # Assertions to check if the number of rounds does not exceed max_round + assert all(len(arr) <= max_round for arr in trip_assistant._oai_messages.values()), "Agent 1 exceeded max rounds" + assert all(len(arr) <= max_round for arr in user_proxy._oai_messages.values()), "User proxy exceeded max rounds" + + +if __name__ == "__main__": + """Runs this file's tests from the command line.""" + test_group_chat_with_llama_index_conversable_agent() diff --git a/website/docs/ecosystem/img/ecosystem-llamaindex.png b/website/docs/ecosystem/img/ecosystem-llamaindex.png new file mode 100644 index 0000000000..ba2ab0a6d1 --- /dev/null +++ b/website/docs/ecosystem/img/ecosystem-llamaindex.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08965da22eddf8c9253e96c1424d9a9b67f210018a3e39372746b93f1d787c04 +size 23548 diff --git a/website/docs/ecosystem/llamaindex.md b/website/docs/ecosystem/llamaindex.md new file mode 100644 index 0000000000..4e9cffeedc --- /dev/null +++ b/website/docs/ecosystem/llamaindex.md @@ -0,0 +1,7 @@ +# Llamaindex + +![Llamaindex Example](img/ecosystem-llamaindex.png) + +[Llamaindex](https://www.llamaindex.ai/) allows the users to create Llamaindex agents and integrate them in autogen conversation patterns. + +- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb)