feat: Update langchain.prompts and langchain_core.messages modules (Q…

…uivrHQ#2326)

This pull request updates the langchain.prompts and
langchain_core.messages modules. It includes changes to the code that
improve functionality and fix any existing issues.

backend/modules/brain/rags/quivr_rag.py

@@ -6,10 +6,10 @@
 from langchain.embeddings.ollama import OllamaEmbeddings
 from langchain.llms.base import BaseLLM
 from langchain.memory import ConversationBufferMemory
-from langchain.prompts import HumanMessagePromptTemplate
+from langchain.prompts import HumanMessagePromptTemplate, SystemMessagePromptTemplate
 from langchain.schema import format_document
 from langchain_community.chat_models import ChatLiteLLM
-from langchain_core.messages import SystemMessage, get_buffer_string
+from langchain_core.messages import get_buffer_string
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate, PromptTemplate
 from langchain_core.runnables import RunnableLambda, RunnablePassthrough
@@ -42,17 +42,21 @@
 Context:
 {context}
 
-User Instructions to follow when answering, default to none: {custom_instructions}
 User Question: {question}
 Answer:
 """
+
+system_message_template = """
+When answering use markdown to make it concise and neat.
+Use the following pieces of context from files provided by the user that are store in a brain to answer  the users question in the same language as the user question. Your name is Quivr. You're a helpful assistant.  
+If you don't know the answer with the context provided from the files, just say that you don't know, don't try to make up an answer.
+User instruction to follow if provided to answer: {custom_instructions}
+"""
+
+
 ANSWER_PROMPT = ChatPromptTemplate.from_messages(
     [
-        SystemMessage(
-            content=(
-                "When answering use markdown or any other techniques to display the content in a nice and aerated way.  Use the following pieces of context from files provided by the user to answer the users question in the same language as the user question. Your name is Quivr. You're a helpful assistant.  If you don't know the answer with the context provided from the files, just say that you don't know, don't try to make up an answer."
-            )
-        ),
+        SystemMessagePromptTemplate.from_template(system_message_template),
         HumanMessagePromptTemplate.from_template(template_answer),
     ]
 )
@@ -246,7 +250,9 @@ def get_chain(self):
         answer = {
             "answer": final_inputs
             | ANSWER_PROMPT
-            | ChatLiteLLM(max_tokens=self.max_tokens, model=self.model, api_base=api_base),
+            | ChatLiteLLM(
+                max_tokens=self.max_tokens, model=self.model, api_base=api_base
+            ),
             "docs": itemgetter("docs"),
         }
 

backend/packages/files/parsers/common.py

@@ -42,9 +42,13 @@ async def process_file(
         for doc in file.documents:  # pyright: ignore reportPrivateUsage=none
             new_metadata = metadata.copy()
             len_chunk = len(enc.encode(doc.page_content))
+            page_content_encoded = doc.page_content.encode("unicode_escape").decode(
+                "ascii", "replace"
+            )
+
             new_metadata["chunk_size"] = len_chunk
             doc_with_metadata = DocumentSerializable(
-                page_content=doc.page_content, metadata=new_metadata
+                page_content=page_content_encoded, metadata=new_metadata
             )
             docs.append(doc_with_metadata)
 

supabase/config.toml

@@ -32,21 +32,25 @@ port = 54329
 # Configure one of the supported pooler modes: `transaction`, `session`.
 pool_mode = "transaction"
 # How many server connections to allow per user/database pair.
-default_pool_size = 15
+default_pool_size = 20
 # Maximum number of client connections allowed.
 max_client_conn = 100
 
 [realtime]
 enabled = true
 # Bind realtime via either IPv4 or IPv6. (default: IPv6)
 # ip_version = "IPv6"
+# The maximum length in bytes of HTTP request headers. (default: 4096)
+# max_header_length = 4096
 
 [studio]
 enabled = true
 # Port to use for Supabase Studio.
 port = 54323
 # External URL of the API server that frontend connects to.
 api_url = "http://localhost"
+# OpenAI API Key to use for Supabase AI in the Supabase Studio.
+openai_api_key = "env(OPENAI_API_KEY)"
 
 # Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
 # are monitored, and you can view the emails that would have been sent from the web interface.
@@ -71,14 +75,16 @@ site_url = "http://localhost:3000"
 # A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
 additional_redirect_urls = ["https://localhost:3000"]
 # How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week).
-jwt_expiry = 604800
+jwt_expiry = 3600
 # If disabled, the refresh token will never expire.
 enable_refresh_token_rotation = true
 # Allows refresh tokens to be reused after expiry, up to the specified interval in seconds.
 # Requires enable_refresh_token_rotation = true.
 refresh_token_reuse_interval = 10
 # Allow/disallow new user signups to your project.
 enable_signup = true
+# Allow/disallow testing manual linking of accounts
+enable_manual_linking = false
 
 [auth.email]
 # Allow/disallow new user signups via email to your project.
@@ -100,12 +106,18 @@ enable_signup = true
 # If enabled, users need to confirm their phone number before signing in.
 enable_confirmations = false
 # Template for sending OTP to users
-template = "Your code is {{ .Code }} ."
+template = "Your code is {{ `{{ .Code }}` }} ."
 
 # Use pre-defined map of phone number to OTP for testing.
 [auth.sms.test_otp]
 # 4152127777 = "123456"
 
+# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used.
+[auth.hook.custom_access_token]
+# enabled = true
+# uri = "pg-functions://<database>/<schema>/<hook_name>"
+
+
 # Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`.
 [auth.sms.twilio]
 enabled = false
@@ -115,7 +127,7 @@ message_service_sid = ""
 auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)"
 
 # Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
-# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin`, `notion`, `twitch`,
+# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`,
 # `twitter`, `slack`, `spotify`, `workos`, `zoom`.
 [auth.external.apple]
 enabled = false
@@ -146,4 +158,4 @@ s3_region = "env(S3_REGION)"
 # Configures AWS_ACCESS_KEY_ID for S3 bucket
 s3_access_key = "env(S3_ACCESS_KEY)"
 # Configures AWS_SECRET_ACCESS_KEY for S3 bucket
-s3_secret_key = "env(S3_SECRET_KEY)"
+s3_secret_key = "env(S3_SECRET_KEY)"