forked from Significant-Gravitas/AutoGPT
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into add_website_memory
- Loading branch information
Showing
17 changed files
with
506 additions
and
187 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster | ||
| ARG VARIANT=3-bullseye | ||
| FROM python:3.8 | ||
|
|
||
| RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ | ||
| # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 | ||
| && apt-get purge -y imagemagick imagemagick-6-common | ||
|
|
||
| # Temporary: Upgrade python packages due to https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-40897 | ||
| # They are installed by the base image (python) which does not have the patch. | ||
| RUN python3 -m pip install --upgrade setuptools | ||
|
|
||
| # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. | ||
| # COPY requirements.txt /tmp/pip-tmp/ | ||
| # RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ | ||
| # && rm -rf /tmp/pip-tmp | ||
|
|
||
| # [Optional] Uncomment this section to install additional OS packages. | ||
| # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ | ||
| # && apt-get -y install --no-install-recommends <your-package-list-here> | ||
|
|
||
| # [Optional] Uncomment this line to install global node packages. | ||
| # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| { | ||
| "build": { | ||
| "dockerfile": "./Dockerfile", | ||
| "context": "." | ||
| }, | ||
| "features": { | ||
| "ghcr.io/devcontainers/features/common-utils:2": { | ||
| "installZsh": "true", | ||
| "username": "vscode", | ||
| "userUid": "1000", | ||
| "userGid": "1000", | ||
| "upgradePackages": "true" | ||
| }, | ||
| "ghcr.io/devcontainers/features/python:1": "none", | ||
| "ghcr.io/devcontainers/features/node:1": "none", | ||
| "ghcr.io/devcontainers/features/git:1": { | ||
| "version": "latest", | ||
| "ppa": "false" | ||
| } | ||
| }, | ||
| // Configure tool-specific properties. | ||
| "customizations": { | ||
| // Configure properties specific to VS Code. | ||
| "vscode": { | ||
| // Set *default* container specific settings.json values on container create. | ||
| "settings": { | ||
| "python.defaultInterpreterPath": "/usr/local/bin/python" | ||
| } | ||
| } | ||
| }, | ||
| // Use 'forwardPorts' to make a list of ports inside the container available locally. | ||
| // "forwardPorts": [], | ||
|
|
||
| // Use 'postCreateCommand' to run commands after the container is created. | ||
| // "postCreateCommand": "pip3 install --user -r requirements.txt", | ||
|
|
||
| // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. | ||
| "remoteUser": "vscode" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,23 @@ | ||
| # Use an official Python base image from the Docker Hub | ||
| FROM python:3.11-slim | ||
| ENV PIP_NO_CACHE_DIR=yes | ||
| WORKDIR /app | ||
| COPY requirements.txt . | ||
| RUN pip install -r requirements.txt | ||
| COPY scripts/ . | ||
| ENTRYPOINT ["python", "main.py"] | ||
|
|
||
| # Set environment variables | ||
| ENV PIP_NO_CACHE_DIR=yes \ | ||
| PYTHONUNBUFFERED=1 \ | ||
| PYTHONDONTWRITEBYTECODE=1 | ||
|
|
||
| # Create a non-root user and set permissions | ||
| RUN useradd --create-home appuser | ||
| WORKDIR /home/appuser | ||
| RUN chown appuser:appuser /home/appuser | ||
| USER appuser | ||
|
|
||
| # Copy the requirements.txt file and install the requirements | ||
| COPY --chown=appuser:appuser requirements.txt . | ||
| RUN pip install --no-cache-dir --user -r requirements.txt | ||
|
|
||
| # Copy the application files | ||
| COPY --chown=appuser:appuser scripts/ . | ||
|
|
||
| # Set the entrypoint | ||
| ENTRYPOINT ["python", "main.py"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import argparse | ||
| import logging | ||
| from config import Config | ||
| from memory import get_memory | ||
| from file_operations import ingest_file, search_files | ||
|
|
||
| cfg = Config() | ||
|
|
||
|
|
||
| def configure_logging(): | ||
| logging.basicConfig(filename='log-ingestion.txt', | ||
| filemode='a', | ||
| format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', | ||
| datefmt='%H:%M:%S', | ||
| level=logging.DEBUG) | ||
| return logging.getLogger('AutoGPT-Ingestion') | ||
|
|
||
|
|
||
| def ingest_directory(directory, memory, args): | ||
| """ | ||
| Ingest all files in a directory by calling the ingest_file function for each file. | ||
| :param directory: The directory containing the files to ingest | ||
| :param memory: An object with an add() method to store the chunks in memory | ||
| """ | ||
| try: | ||
| files = search_files(directory) | ||
| for file in files: | ||
| ingest_file(file, memory, args.max_length, args.overlap) | ||
| except Exception as e: | ||
| print(f"Error while ingesting directory '{directory}': {str(e)}") | ||
|
|
||
|
|
||
| def main(): | ||
| logger = configure_logging() | ||
|
|
||
| parser = argparse.ArgumentParser(description="Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script.") | ||
| group = parser.add_mutually_exclusive_group(required=True) | ||
| group.add_argument("--file", type=str, help="The file to ingest.") | ||
| group.add_argument("--dir", type=str, help="The directory containing the files to ingest.") | ||
| parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content (default: False)", default=False) | ||
| parser.add_argument("--overlap", type=int, help="The overlap size between chunks when ingesting files (default: 200)", default=200) | ||
| parser.add_argument("--max_length", type=int, help="The max_length of each chunk when ingesting files (default: 4000)", default=4000) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| # Initialize memory | ||
| memory = get_memory(cfg, init=args.init) | ||
| print('Using memory of type: ' + memory.__class__.__name__) | ||
|
|
||
| if args.file: | ||
| try: | ||
| ingest_file(args.file, memory, args.max_length, args.overlap) | ||
| print(f"File '{args.file}' ingested successfully.") | ||
| except Exception as e: | ||
| logger.error(f"Error while ingesting file '{args.file}': {str(e)}") | ||
| print(f"Error while ingesting file '{args.file}': {str(e)}") | ||
| elif args.dir: | ||
| try: | ||
| ingest_directory(args.dir, memory, args) | ||
| print(f"Directory '{args.dir}' ingested successfully.") | ||
| except Exception as e: | ||
| logger.error(f"Error while ingesting directory '{args.dir}': {str(e)}") | ||
| print(f"Error while ingesting directory '{args.dir}': {str(e)}") | ||
| else: | ||
| print("Please provide either a file path (--file) or a directory name (--dir) inside the auto_gpt_workspace directory as input.") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
Oops, something went wrong.