Merge branch 'main' into feat/document_ie

.drone.yml

@@ -0,0 +1,42 @@
+kind: pipeline # 定义对象类型，还有secret和signature两种类型
+type: docker # 定义流水线类型，还有kubernetes、exec、ssh等类型
+name: cicd # 定义流水线名称
+
+steps: # 定义流水线执行步骤，这些步骤将顺序执行
+  - name: package # 流水线名称
+    image: python:3.10-slim # 定义创建容器的Docker镜像
+    volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
+      - name: maven-cache
+        path: /root/.m2 # 将maven下载依赖的目录挂载出来，防止重复下载
+      - name: maven-build
+        path: /app/build # 将应用打包好的Jar和执行脚本挂载出来
+    commands: # 定义在Docker容器中执行的shell命令
+      - pip install Cython
+      - pip install wheel
+      - pip install twine
+      - cd ./src/bisheng-langchain
+      - python setup.py bdist_wheel
+      - cp dist.* /app/build/
+
+  - name: build_backend
+    image: python:3.10-slim # 定义创建容器的Docker镜像
+    volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
+      - name: maven-cache
+        path: /root/.m2 # 将maven下载依赖的目录挂载出来，防止重复下载
+      - name: maven-build
+        path: /app/build # 将应用打包好的Jar和执行脚本挂载出来
+    commands:
+      - cd ./src/backend
+      - pip install bisheng_langchain==$RELEASE_VERSION
+      - sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
+      - poetry lock
+
+  - name: build-image # 步骤名称
+    image: plugins/docker # 使用镜像
+    settings: # 当前设置
+      username: # 账号名称
+        from_secret: docker_username
+      password: # 账号密码
+        from_secret: docker_password
+      dockerfile: deploy/Dockerfile # Dockerfile地址， 注意是相对地址
+      repo: yxs970707/deploy-web-demo # 镜像名称

.github/workflows/ci.yml

@@ -132,6 +132,7 @@ jobs:
         id: process_message
         run: |
           value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
+          value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
           echo "message=${value}" >> $GITHUB_ENV
         shell: bash
 

.github/workflows/release.yml

@@ -24,31 +24,20 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2 
 
-      - name: git message
-        id: git_message
-        run: |
-          echo "::set-output name=message::$( echo '${{github.event.head_commit.message}}' | sed ':a;N;$!ba;s/\n/\\n/g')"
-
       - name: Process git message
         id: process_message
         run: |
-          message=""
-          IFS=$'\n'
-          for line in $(echo "${{ steps.git_message.outputs.message }}"); do
-            message+="$line\n"
-          done
-          echo "::set-output name=message::$message"
+          value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/\\\\n/g')
+          echo "message=${value}" >> $GITHUB_ENV
         shell: bash
 
       - name: notify feishu
         uses: fjogeleit/http-request-action@v1
         with:
           url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
           method: 'POST'
-          data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译",  "content": [[{"tag":"text","text":"提交内容${{ steps.process_message.outputs.message }}"}]]}}}}'
-
-
-      # 构建 bisheng-langchain
+          data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译",  "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
+
       - name: Set python version 3.8
         uses: actions/setup-python@v1
         with:

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: ^scripts|docs|docker|requirements|README.md|test
+exclude: ^scripts|docs|docker|requirements|README.md|test|experimental
 repos:
   - repo: https://github.com/PyCQA/flake8.git
     rev: 3.8.3

docker/docker-compose-model.yml

@@ -1,7 +1,7 @@
 services:
   bisheng-rt:
     container_name: bisheng-rt
-    image: dataelement/bisheng-rt:0.0.6.2
+    image: dataelement/bisheng-rt:0.0.6.3rc1
     shm_size: 10gb
     ports:
       - "9000:9000"
@@ -16,7 +16,10 @@ services:
               device_ids: ['0,1']  # 指定想映射给rt服务使用的宿主机上的GPU ID号，如想映射多个卡，可写为['0','1','2']
     environment:
       TZ: Asia/Shanghai
+    # 不使用闭源模型的话，用下面的启动命令
     command: ["./bin/rtserver", "f"]
+    # 使用闭源模型的话，用下面的启动命令，地址替换为授权地址
+    # command: ["bash", "bin/entrypoint.sh", "--serveraddr=<license srv host>"]
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/data/llm:/opt/bisheng-rt/models/model_repository  # 冒号前为宿主机上放置模型目录的路径，请根据实际环境修改；冒号后为映射到容器内的路径，请勿修改
     healthcheck:
@@ -56,7 +59,7 @@ services:
 
   bisheng-unstructured:
     container_name: bisheng-unstructured
-    image: dataelement/bisheng-unstructured:0.0.3.2
+    image: dataelement/bisheng-unstructured:0.0.3.4
     ports:
       - "10001:10001"
     environment:

docker/docker-compose.yml

@@ -32,7 +32,7 @@ services:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/redis/redis.conf:/etc/redis.conf
     command: redis-server /etc/redis.conf
     healthcheck:
-      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      test: ["CMD-SHELL", 'redis-cli ping|grep -e "PONG\|NOAUTH"']
       interval: 10s
       timeout: 5s
       retries: 3
@@ -45,6 +45,7 @@ services:
       - "8701:80"
     environment:
       TZ: Asia/Shanghai
+      JWT_ENABLED: false
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/office/bisheng:/var/www/onlyoffice/documentserver/sdkjs-plugins/bisheng
     command: bash -c "supervisorctl restart all"
@@ -94,7 +95,7 @@ services:
 
   elasticsearch:
     container_name: bisheng-es
-    image: docker.io/bitnami/elasticsearch:8
+    image: docker.io/bitnami/elasticsearch:8.12.0
     user: root
     ports:
       - "9200:9200"

src/backend/Dockerfile

@@ -5,6 +5,8 @@ WORKDIR /app
 # Install Poetry
 RUN apt-get update && apt-get install gcc g++ curl build-essential postgresql-server-dev-all -y
 RUN apt-get update && apt-get install procps -y
+# Install font
+RUN apt install fonts-wqy-zenhei -y
 # opencv
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
 RUN curl -sSL https://install.python-poetry.org | python3 -

src/backend/bisheng/__init__.py

@@ -1,6 +1,7 @@
 from importlib import metadata
 
 from bisheng.cache import cache_manager  # noqa: E402
+from bisheng.interface.custom.custom_component import CustomComponent
 from bisheng.processing.process import load_flow_from_json  # noqa: E402
 
 try:
@@ -11,4 +12,4 @@
 del metadata  # optional, avoids polluting the results of dir(__package__)
 
 
-__all__ = ['load_flow_from_json', 'cache_manager']
+__all__ = ['load_flow_from_json', 'cache_manager', 'CustomComponent']

src/backend/bisheng/api/errcode/assistant.py

@@ -0,0 +1,22 @@
+from bisheng.api.errcode.base import BaseErrorCode
+
+
+# 组件模块 返回错误码，业务代码104
+class AssistantNotExistsError(BaseErrorCode):
+    Code: int = 10400
+    Msg: str = '助手不存在'
+
+
+class AssistantInitError(BaseErrorCode):
+    Code: int = 10401
+    Msg: str = '助手上线失败'
+
+
+class AssistantNameRepeatError(BaseErrorCode):
+    Code: int = 10402
+    Msg: str = '助手名称重复'
+
+
+class AssistantNotEditError(BaseErrorCode):
+    Code: int = 10403
+    Msg: str = '助手已上线，不可编辑'

src/backend/bisheng/api/errcode/base.py

@@ -9,3 +9,8 @@ class BaseErrorCode:
     @classmethod
     def return_resp(cls, msg: str = None, data: any = None) -> UnifiedResponseModel:
         return UnifiedResponseModel(status_code=cls.Code, status_message=msg or cls.Msg, data=data)
+
+
+class UnAuthorizedError(BaseErrorCode):
+    Code: int = 403
+    Msg: str = '暂无操作权限'

src/backend/bisheng/api/errcode/component.py

@@ -0,0 +1,12 @@
+from bisheng.api.errcode.base import BaseErrorCode
+
+
+# 组件模块 返回错误码，业务代码103
+class ComponentExistError(BaseErrorCode):
+    Code: int = 10300
+    Msg: str = '组件已存在'
+
+
+class ComponentNotExistError(BaseErrorCode):
+    Code: int = 10301
+    Msg: str = '组件不存在'

src/backend/bisheng/api/router.py

@@ -1,7 +1,8 @@
 # Router for base api
-from bisheng.api.v1 import (chat_router, endpoints_router, finetune_router, flows_router,
-                            knowledge_router, qa_router, report_router, server_router,
-                            skillcenter_router, user_router, validate_router, variable_router)
+from bisheng.api.v1 import (assistant_router, chat_router, component_router, endpoints_router,
+                            finetune_router, flows_router, knowledge_router, qa_router,
+                            report_router, server_router, skillcenter_router, user_router,
+                            validate_router, variable_router)
 from bisheng.api.v2 import chat_router_rpc, knowledge_router_rpc, rpc_router_rpc
 from fastapi import APIRouter
 
@@ -18,6 +19,8 @@
 router.include_router(variable_router)
 router.include_router(report_router)
 router.include_router(finetune_router)
+router.include_router(component_router)
+router.include_router(assistant_router)
 
 router_rpc = APIRouter(prefix='/api/v2', )
 router_rpc.include_router(knowledge_router_rpc)