Merge pull request #3 from a232319779/fix-package

[FIX]补充scrapy依赖的库；自动生成scrapy.cfg文件

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 ddvv
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

NextBSpiders/cli/telegram_run_spider.py

@@ -11,12 +11,29 @@
 NextBSpider执行telegram爬虫命令行工具
 """
 
+import os
 import argparse
 import json
 import base64
 from scrapy import cmdline
 from NextBSpiders.libs.nextb_spier_db import NextBTGSQLITEDB
 
+scrapy_cfg = """# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = NextBSpiders.settings
+"""
+
+def process_scrapy_cfg_file():
+    current_dir = os.path.abspath(".")
+    scrapy_file = os.path.join(current_dir, "scrapy.cfg")
+    if not os.path.exists(scrapy_file):
+        with open(scrapy_file, "w") as f:
+            f.write(scrapy_cfg)
+            f.flush()
 
 def parse_cmd():
     """
@@ -63,6 +80,7 @@ def telegram_run_spider(config_file):
         param_base64=param_base64,
         db_name=config_js.get("sqlite_db_name", "tg_sqlite.db"),
     )
+    process_scrapy_cfg_file()
     cmdline.execute(cmd.split())
 
 

README.md

@@ -66,4 +66,23 @@ pip install NextBSpiders
 
 ## 注意事项
 
-1. 每个telegram爬虫配置文件中的`session_name`和`sqlite_db_name`需要独立使用，避免出现`sqlite`数据库读写暂用问题
+1. 每个telegram爬虫配置文件中的`session_name`和`sqlite_db_name`需要独立使用，避免出现`sqlite`数据库读写暂用问题
+2. 默认执行`nextb-telegram-run-spider`命令时，会在当前目录创建一个`scrapy.cfg`文件用以保证`scrapy`爬虫的执行环境。不排除在某些情况下，可能需要**在执行`nextb-telegram-run-spider.exe -c $config.json`的目录下，手动创建一个`scrapy.cfg`文件**
+
+## scrapy.cfg
+
+scrapy.cfg内容如下，避免执行`scrapy`出错
+
+```
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = NextBSpiders.settings
+```
+
+## 数据库示例
+
+![](https://github.com/a232319779/NextBSpiders/blob/master/pictures/example.png)

requirements.txt

@@ -1,5 +1,5 @@
+pyOpenSSL==22.0.0
 Scrapy==2.6.1
-setuptools==58.0.4
 SQLAlchemy==1.4.31
 Telethon==1.24.0
 Twisted==22.4.0

setup.py

@@ -19,6 +19,8 @@ def do_setup(**kwargs):
 
 long_description = """
 
+# NextB 爬虫
+
 基于scrapy框架的爬虫项目
 
 ## 准备
@@ -86,11 +88,29 @@ def do_setup(**kwargs):
 ## 注意事项
 
 1. 每个telegram爬虫配置文件中的`session_name`和`sqlite_db_name`需要独立使用，避免出现`sqlite`数据库读写暂用问题
+2. 默认执行`nextb-telegram-run-spider`命令时，会在当前目录创建一个`scrapy.cfg`文件用以保证`scrapy`爬虫的执行环境。不排除在某些情况下，可能需要**在执行`nextb-telegram-run-spider.exe -c $config.json`的目录下，手动创建一个`scrapy.cfg`文件**
+
+## scrapy.cfg
+
+scrapy.cfg内容如下，避免执行`scrapy`出错
+
+```
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = NextBSpiders.settings
+```
+
+## 数据库示例
+
 """
 
 do_setup(
     name="NextBSpiders",
-    version="1.0.0",
+    version="1.0.1",
     author="ddvv",
     author_email="[email protected]",
     description="基于scrapy的telegram爬虫",
@@ -112,13 +132,13 @@ def do_setup(**kwargs):
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],
-    python_requires=">=3.6",
+    python_requires=">=3.6,<3.9",
     keywords=[],
     license="MIT",
     include_package_data=True,
     install_requires=[
+        "pyOpenSSL==22.0.0",
         "Scrapy==2.6.1",
-        "setuptools==58.0.4",
         "SQLAlchemy==1.4.31",
         "Telethon==1.24.0",
         "Twisted==22.4.0",