update deps

README.md

@@ -5,7 +5,9 @@ Multi-Agent Programming framework to build LLM applications.
 
 ## Set up virtual env and install `langroid`
 
-IMPORTANT: Please ensure you are using Python 3.11+
+IMPORTANT: Please ensure you are using Python 3.11+. If you are using poetry,
+you may be able to just run `poetry env use 3.11` if you have Python 3.11 available in your system.
+
 
 ```bash
 # clone the repo and cd into repo root
@@ -18,11 +20,14 @@ python3 -m venv .venv
 # activate the virtual env
 . .venv/bin/activate
 
-# install with `hf-embeddings` extra to be able to use sentence_transformers embeddings
-pip install "langroid[hf-embeddings]"
+# install dependencies from pyproject.toml:
+# This install langroid with extras.
+poetry install 
+# or equivalently:
+# pip install "langroid[hf-embeddings,postgres,mysql]"
 
 # or to update an existing installation:
-pip install --upgrade "langroid[hf-embeddings]"
+pip install --upgrade "langroid[hf-embeddings,postgres,mysql]"
 ```
 
 

examples/basic/chat-search.py

@@ -17,7 +17,6 @@
 
 from langroid.agent.chat_agent import ChatAgent, ChatAgentConfig
 from langroid.agent.task import Task
-from langroid.language_models.base import LocalModelConfig
 from langroid.agent.tools.google_search_tool import GoogleSearchTool
 from langroid.language_models.openai_gpt import OpenAIGPTConfig
 from langroid.utils.configuration import set_global, Settings
@@ -28,15 +27,29 @@
 
 setup_colored_logging()
 
+# create classes for other model configs
+LiteLLMOllamaConfig = OpenAIGPTConfig.create(prefix="ollama")
+litellm_ollama_config = LiteLLMOllamaConfig(
+    chat_model="ollama/llama2",
+    completion_model="ollama/llama2",
+    api_base="http://localhost:11434",
+    litellm=True,
+    chat_context_length=4096,
+    use_completion_for_chat=False,
+)
+OobaConfig = OpenAIGPTConfig.create(prefix="ooba")
+ooba_config = OobaConfig(
+    chat_model="local",  # doesn't matter
+    completion_model="local",  # doesn't matter
+    api_base="http://localhost:8000/v1",  # <- edit if running at a different port
+    chat_context_length=2048,
+    litellm=False,
+    use_completion_for_chat=False,
+)
+
 
 class CLIOptions(BaseSettings):
-    local: bool = False
-    api_base: str = "http://localhost:8000/v1"
-    local_model: str = ""
-    local_ctx: int = 2048
-    # use completion endpoint for chat?
-    # if so, we should format chat->prompt ourselves, if we know the required syntax
-    completion: bool = False
+    model: str = ""
 
     class Config:
         extra = "forbid"
@@ -60,24 +73,13 @@ def chat(opts: CLIOptions) -> None:
 
     load_dotenv()
 
-    # create the appropriate OpenAIGPTConfig depending on local model or not
-
-    if opts.local or opts.local_model:
-        # assumes local endpoint is either the default http://localhost:8000/v1
-        # or if not, it has been set in the .env file as the value of
-        # OPENAI_LOCAL.API_BASE
-        local_model_config = LocalModelConfig(
-            api_base=opts.api_base,
-            model=opts.local_model,
-            context_length=opts.local_ctx,
-            use_completion_for_chat=opts.completion,
-        )
-        llm_config = OpenAIGPTConfig(
-            local=local_model_config,
-            timeout=180,
-        )
+    # use the appropriate config instance depending on model name
+    if opts.model == "ooba":
+        llm_config = ooba_config
+    elif opts.model.startswith("ollama"):
+        llm_config = litellm_ollama_config
+        llm_config.chat_model = opts.model
     else:
-        # defaults to chat_model = OpenAIChatModel.GPT4
         llm_config = OpenAIGPTConfig()
 
     config = ChatAgentConfig(
@@ -110,31 +112,19 @@ def chat(opts: CLIOptions) -> None:
         """,
     )
     # local models do not like the first message to be empty
-    user_message = "Hello." if (opts.local or opts.local_model) else None
+    user_message = "Hello." if (opts.model != "") else None
     task.run(user_message)
 
 
 @app.command()
 def main(
     debug: bool = typer.Option(False, "--debug", "-d", help="debug mode"),
+    model: str = typer.Option("", "--model", "-m", help="model name"),
     no_stream: bool = typer.Option(False, "--nostream", "-ns", help="no streaming"),
     nocache: bool = typer.Option(False, "--nocache", "-nc", help="don't use cache"),
     cache_type: str = typer.Option(
         "redis", "--cachetype", "-ct", help="redis or momento"
     ),
-    local: bool = typer.Option(False, "--local", "-l", help="use local llm"),
-    local_model: str = typer.Option(
-        "", "--local_model", "-lm", help="local model path"
-    ),
-    api_base: str = typer.Option(
-        "http://localhost:8000/v1", "--api_base", "-api", help="local model api base"
-    ),
-    local_ctx: int = typer.Option(
-        2048, "--local_ctx", "-lc", help="local llm context size (default 2048)"
-    ),
-    completion: bool = typer.Option(
-        False, "--completion", "-c", help="use completion endpoint for chat"
-    ),
 ) -> None:
     set_global(
         Settings(
@@ -144,13 +134,7 @@ def main(
             cache_type=cache_type,
         )
     )
-    opts = CLIOptions(
-        local=local,
-        api_base=api_base,
-        local_model=local_model,
-        local_ctx=local_ctx,
-        completion=completion,
-    )
+    opts = (CLIOptions(model=model),)
     chat(opts)
 
 

examples/basic/chat.py

@@ -8,15 +8,23 @@
 
 Use optional arguments to change the settings, e.g.:
 
--l # use locally running Llama model
--lc 1000 # use local Llama model with context length 1000
+-m "local" to use a model served locally at an OpenAI-API-compatible endpoint
+[ Ensure the API endpoint url matches the one in the code below, or edit it. ]
+OR
+- m "litellm/ollama/llama2" to use any model supported by litellm
+(see list here https://docs.litellm.ai/docs/providers)
+[Note you must prepend "litellm/" to the model name required in the litellm docs,
+e.g. "ollama/llama2" becomes "litellm/ollama/llama2",
+"bedrock/anthropic.claude-instant-v1" becomes
+"litellm/bedrock/anthropic.claude-instant-v1"]
+
 -ns # no streaming
 -d # debug mode
 -nc # no cache
 -ct momento # use momento cache (instead of redis)
 
 For details on running with local Llama model, see:
-https://langroid.github.io/langroid/tutorials/llama/
+https://langroid.github.io/langroid/blog/2023/09/14/using-langroid-with-local-llms/
 """
 import typer
 from rich import print
@@ -26,7 +34,6 @@
 
 from langroid.agent.chat_agent import ChatAgent, ChatAgentConfig
 from langroid.agent.task import Task
-from langroid.language_models.base import LocalModelConfig
 from langroid.language_models.openai_gpt import OpenAIGPTConfig
 from langroid.utils.configuration import set_global, Settings
 from langroid.utils.logging import setup_colored_logging
@@ -36,12 +43,49 @@
 
 setup_colored_logging()
 
+# Create classes for non-OpenAI model configs
+
+# OPTION 1: LiteLLM-supported models
+# -----------------------------------
+# For any model supported by litellm
+# (see list here https://docs.litellm.ai/docs/providers)
+# The `chat_model` should be specified as "litellm/" followed by
+# the chat_model name in the litellm docs.
+# In this case Langroid uses the litellm adapter library to
+# translate between OpenAI API and the model's API.
+# For external (remote) models, typical there will be specific env vars
+# (e.g. API Keys, etc) that need to be set.
+# If those are not set, you will get an err msg saying which vars need to be set.
+
+# OPTION 2: Local models served at an OpenAI-compatible API endpoint
+# -----------------------------------------------------------------
+# Use this config for any model that is locally served at an
+# OpenAI-compatible API endpoint, for example, using either the
+# litellm proxy server https://docs.litellm.ai/docs/proxy_server
+# or the oooba/text-generation-webui server
+# https://github.com/oobabooga/text-generation-webui/tree/main/extensions/openai
+#
+# In this case the `chat_model` name should be specified as
+# "local/localhost:8000/v1" or "local/localhost:8000" or other port number
+# depending on how you launch the model locally.
+# Langroid takes care of extracting the local URL to set the `api_base`
+# of the config so that the `openai.*` completion functions can be used
+# without having to rely on adapter libraries like litellm.
+
+MyLLMConfig = OpenAIGPTConfig.create(prefix="myllm")
+my_llm_config = MyLLMConfig(
+    chat_model="litellm/ollama/llama2",
+    # or, other possibilities for example:
+    # "litellm/bedrock/anthropic.claude-instant-v1"
+    # "litellm/ollama/llama2"
+    # "local/localhost:8000/v1"
+    # "local/localhost:8000"
+    chat_context_length=2048,  # adjust based on model
+)
+
 
 class CLIOptions(BaseSettings):
-    local: bool = False
-    api_base: str = "http://localhost:8000/v1"
-    local_model: str = ""
-    local_ctx: int = 2048
+    model: str = ""
 
     class Config:
         extra = "forbid"
@@ -58,22 +102,13 @@ def chat(opts: CLIOptions) -> None:
 
     load_dotenv()
 
-    # create the appropriate OpenAIGPTConfig depending on local model or not
+    # use the appropriate config instance depending on model name
+    if opts.model.startswith("litellm/") or opts.model.startswith("local/"):
+        # e.g. litellm/ollama/llama2 or litellm/bedrock/anthropic.claude-instant-v1
+        llm_config = my_llm_config
+        llm_config.chat_model = opts.model
 
-    if opts.local or opts.local_model:
-        # assumes local endpoint is either the default http://localhost:8000/v1
-        # or if not, it has been set in the .env file as the value of
-        # OPENAI_LOCAL.API_BASE
-        local_model_config = LocalModelConfig(
-            api_base=opts.api_base,
-            model=opts.local_model,
-            context_length=opts.local_ctx,
-        )
-        llm_config = OpenAIGPTConfig(
-            local=local_model_config,
-        )
     else:
-        # defaults to chat_model = OpenAIChatModel.GPT4
         llm_config = OpenAIGPTConfig()
 
     default_sys_msg = "You are a helpful assistant. Be concise in your answers."
@@ -86,27 +121,20 @@ def chat(opts: CLIOptions) -> None:
     config = ChatAgentConfig(
         system_message=sys_msg,
         llm=llm_config,
-        vecdb=None,
     )
     agent = ChatAgent(config)
     task = Task(agent)
-    user_message = "Hello." if (opts.local or opts.local_model) else None
-    task.run(user_message)
+    # OpenAI models are ok with just a system msg,
+    # but in some scenarios, other (e.g. llama) models
+    # seem to do better when kicked off with a sys msg and a user msg.
+    # In those cases we may want to do task.run("hello") instead.
+    task.run()
 
 
 @app.command()
 def main(
     debug: bool = typer.Option(False, "--debug", "-d", help="debug mode"),
-    local: bool = typer.Option(False, "--local", "-l", help="use local llm"),
-    local_model: str = typer.Option(
-        "", "--local_model", "-lm", help="local model path"
-    ),
-    api_base: str = typer.Option(
-        "http://localhost:8000/v1", "--api_base", "-api", help="local model api base"
-    ),
-    local_ctx: int = typer.Option(
-        2048, "--local_ctx", "-lc", help="local llm context size (default 2048)"
-    ),
+    model: str = typer.Option("", "--model", "-m", help="model name"),
     no_stream: bool = typer.Option(False, "--nostream", "-ns", help="no streaming"),
     nocache: bool = typer.Option(False, "--nocache", "-nc", help="don't use cache"),
     cache_type: str = typer.Option(
@@ -121,12 +149,7 @@ def main(
             cache_type=cache_type,
         )
     )
-    opts = CLIOptions(
-        local=local,
-        api_base=api_base,
-        local_model=local_model,
-        local_ctx=local_ctx,
-    )
+    opts = CLIOptions(model=model)
     chat(opts)
 
 

examples/data-qa/sql-chat/sql_chat.py

@@ -5,7 +5,9 @@
 
 (a) Install PostgreSQL dev libraries for your platform, e.g.
     - `sudo apt-get install libpq-dev` on Ubuntu,
-    - `brew install postgresql` on Mac, etc.
+    - On Mac:
+        - `brew install postgresql`, then:
+        - `brew services start postgresql`
 (b) langroid with the postgres extra, e.g. `pip install langroid[postgres]`
     or `poetry add langroid[postgres]` or `poetry install -E postgres`.
     If this gives you an error, try `pip install psycopg2-binary` in your virtualenv.

examples/docqa/chat-search.py

@@ -61,7 +61,7 @@ def relevant_extracts(self, msg: RelevantExtractsTool) -> str:
         """Get docs/extracts relevant to the query, from vecdb"""
         self.tried_vecdb = True
         query = msg.query
-        extracts = self.get_relevant_extracts(query)
+        _, extracts = self.get_relevant_extracts(query)
         if len(extracts) == 0:
             return NO_ANSWER
         return "\n".join(str(e) for e in extracts)
@@ -77,12 +77,13 @@ def relevant_search_extracts(self, msg: RelevantSearchExtractsTool) -> str:
         links = [r.link for r in results]
         self.config.doc_paths = links
         self.ingest()
-        extracts = self.get_relevant_extracts(query)
+        _, extracts = self.get_relevant_extracts(query)
         return "\n".join(str(e) for e in extracts)
 
 
 class CLIOptions(BaseSettings):
     fn_api: bool = False
+    model: str = ""
 
 
 def chat(opts: CLIOptions) -> None:
@@ -140,10 +141,22 @@ def chat(opts: CLIOptions) -> None:
     agent = GoogleSearchDocChatAgent(config)
     agent.enable_message(RelevantExtractsTool)
     agent.enable_message(RelevantSearchExtractsTool)
-    collection_name = "docqa-chat-search"
-    print(f"[red]Using {collection_name}, possibly replacing it")
+    collection_name = Prompt.ask(
+        "Name a collection to use",
+        default="docqa-chat-search",
+    )
+    replace = (
+        Prompt.ask(
+            "Would you like to replace this collection?",
+            choices=["y", "n"],
+            default="n",
+        )
+        == "y"
+    )
+
+    print(f"[red]Using {collection_name}")
 
-    agent.vecdb.set_collection(collection_name, replace=True)
+    agent.vecdb.set_collection(collection_name, replace=replace)
 
     task = Task(
         agent,
@@ -157,13 +170,15 @@ def chat(opts: CLIOptions) -> None:
 def main(
     debug: bool = typer.Option(False, "--debug", "-d", help="debug mode"),
     nocache: bool = typer.Option(False, "--nocache", "-nc", help="don't use cache"),
+    model: str = typer.Option("", "--model", "-m", help="model name"),
     fn_api: bool = typer.Option(False, "--fn_api", "-f", help="use functions api"),
     cache_type: str = typer.Option(
         "redis", "--cachetype", "-ct", help="redis or momento"
     ),
 ) -> None:
     cli_opts = CLIOptions(
         fn_api=fn_api,
+        model=model,
     )
 
     set_global(