init

.gitattributes

@@ -0,0 +1,3 @@
+*.js linguist-language=python
+*.css linguist-language=python
+*.html linguist-language=python

.gitignore

@@ -0,0 +1,102 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+.idea/
+bin/
+spider_log.txt
+dump.rdb
+.DS_Store
+cover/
+celerybeat.pid
+oj.py
+/webspider/log
+/webspider/security_constants.py
+celerybeat-schedule
+cove
+nohup.out

.landscape.yml

@@ -0,0 +1,32 @@
+autodetect: yes
+test-warnings: true
+doc-warnings: true
+strictness: veryhigh
+max-line-length: 120
+python-targets: 3
+
+uses:
+  - celery
+
+ignore-paths:
+  - .git
+  - coverage
+  - env
+  - test
+  - webspider/web/templates
+  - webspider/web/static
+
+pep8:
+  run: true
+  disable:
+    - W291
+    - E501
+
+pyflakes:
+    run: true
+
+inherits: [flake8]
+
+requirements:
+  - requirements.txt
+

.travis.yml

@@ -0,0 +1,22 @@
+language: python
+
+sudo: false
+
+python:
+  - "3.6"
+
+services:
+  - mysql
+  - redis-server
+
+before_install:
+  - mysql -e 'CREATE DATABASE spider CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'
+
+install:
+  - make
+
+script:
+  - make test
+
+after_success:
+  - env/bin/codecov

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,25 @@
+PYTHON:=$(shell which python3)
+
+all: python
+
+.PHONY: clean python test flake8
+
+python: setup.py requirements.txt
+	pip install virtualenv
+	echo "\n Creating python virtual environment......\n"
+	virtualenv -p $(PYTHON) env
+	echo "\n Use python virtual environment to install required packages......\n"
+	env/bin/pip install -e .
+	mkdir -p webspider/log
+	touch webspider/log/spider_log.txt
+
+test: flake8
+	env/bin/nosetests -vd
+
+flake8:
+	env/bin/flake8
+
+clean:
+	-rm -rf env cover *eggs *.egg-info *.egg webspider/log
+	@find . -type f -name "*.py[co]" -delete
+	@find . -type d -name "__pycache__" -delete

README.md

@@ -0,0 +1,105 @@
+#
+
+[![Build Status](https://travis-ci.org/JustForFunnnn/webspider.svg)](https://travis-ci.org/JustForFunnnn/webspider)
+[![codecov](https://codecov.io/gh/JustForFunnnn/webspider/branch/master/graph/badge.svg)](https://codecov.io/gh/JustForFunnnn/webspider)
+[![Code Health](https://landscape.io/github/JustForFunnnn/webspider/master/landscape.svg?style=flat)](https://landscape.io/github/JustForFunnnn/webspider/master)
+[![License](https://img.shields.io/github/license/JustForFunnnn/webspider.svg)](https://github.com/JustForFunnnn/webspider/blob/master/LICENSE)
+[![Python](https://img.shields.io/badge/python-3-ff69b4.svg)](https://github.com/JustForFunnnn/webspider)
+
+| --       | --                                         |
+| -------- | ------------------------------------------ |
+| Version  | 1.0.1                                      |
+| WebSite  | http://119.23.223.90:8000                  |
+| Source   | https://github.com/JustForFunnnn/webspider |
+| Keywords | `Python3`, `Tornado`, `Celery`, `Requests` |
+
+## 项目简介
+
+本项目使用的编程语言是`python3`，数据库用的是`MySQL`, 主要用到的库是`celery`和`requests`，并实现了定时任务，出错重试，日志记录，自动更改`Cookies`等的功能，使用`ECharts` + `Bootstrap` 来构建前端页面。
+
+## 展示页面
+
+![Alt text](job-chart.jpg)
+
+## Quick Start
+> 以下操作均是在 `Linux - Ubuntu` 环境下执行
+
+* 克隆项目
+
+```bash
+git clone [email protected]:JustForFunnnn/webspider.git
+```
+
+* 安装 `MySQL`, `Redis`, `Python3`
+
+```bash
+# 安装 redis
+apt-get install redis-server
+
+# 后台启动 redis-server
+nohup redis-server &
+
+# 安装 python3
+apt-get install python3
+
+# 安装 MySQL
+apt-get install mysql-server
+
+# 启动 MySQL
+sudo service mysql start
+```
+
+* 配置数据库和表
+```mysql
+# 创建数据库
+CREATE DATABASE `spider` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+# 还需要创建相关表，表的定义语句在 tests/schema.sql 文件中，可自行复制进 MySQL 命令行中执行。
+```
+
+* 在项目根目录下构建
+```bash
+make
+# 构建成功后项目的 env/bin 目录下会有可执行脚本
+```
+
+* 执行单元测试
+```bash
+make test
+```
+
+* 代码风格检查
+```bash
+make flake8
+```
+
+* 运行 `Web Server`
+```bash
+env/bin/web
+```
+
+* 运行爬虫程序
+```bash
+# 启动定时任务分发器
+env/bin/celery_beat
+# 启动爬取 数据 的 worker
+env/bin/celery_lg_data_worker
+# 启动爬取 职位数据 的 worker
+env/bin/celery_lg_jobs_data_worker
+# 启动爬取 职位数量 的 worker
+env/bin/celery_lg_jobs_count_worker
+```
+
+* env/bin 目录下其他可执行脚本
+```bash
+# 直接爬取职位数量
+env/bin/crawl_lg_jobs_count
+# 直接爬取职位数据
+env/bin/crawl_lg_data
+# 启动celery监控
+env/bin/celery_flower
+```
+
+* 清除构建信息
+```bash
+make clean
+```

requirements.txt

@@ -0,0 +1,21 @@
+# All requirements put in this file
+# You **MUST** specify the package version in this file
+
+tornado == 4.5.3
+gevent == 1.2.2
+gunicorn == 19.7.1
+lxml == 4.1.0
+requests == 2.18.4
+mysqlclient == 1.3.12
+sqlalchemy == 1.2.2
+redis == 2.10.6
+python-redis == 0.1.7
+retrying == 1.3.3
+celery == 4.0.2
+
+flower == 0.9.2
+ipython == 6.2.1
+nose == 1.3.7
+coverage == 4.4.2
+flake8 == 3.5.0
+codecov == 2.0.15

setup.cfg

@@ -0,0 +1,17 @@
+[flake8]
+ignore = W291
+max-line-length = 120
+exclude =
+    .git,
+    eggs,
+    env,
+    tests
+
+[nosetests]
+logging-clear-handlers = 1
+with-coverage = 1
+cover-package = webspider
+cover-erase = 1
+logging-level = DEBUG
+cover-xml = 1
+cover-xml-file = coverage.xml