-
Notifications
You must be signed in to change notification settings - Fork 1
/
models.py
87 lines (65 loc) · 2.5 KB
/
models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
from coagg import db
import requests
import re
import urllib
import sys
from datetime import datetime
from .functions import fetch_all
class Message(db.Model):
__tablename__ = 'messages'
id = db.Column(db.Integer, primary_key=True)
message = db.Column(db.Text)
created = db.Column(db.DateTime, default=datetime.utcnow)
class Comic(db.Model):
__tablename__ = 'comics'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text)
base_url = db.Column(db.Text)
img_url = db.Column(db.Text)
@staticmethod
def update_all_links(data):
def parse_pages(base_urls, patterns, combine_base):
pages = fetch_all(base_urls)
results = []
for base_url, pattern, page, cb in zip(base_urls, patterns, pages, combine_base):
match = re.search(pattern, page)
try:
if cb:
results.append(base_url + match.group(1))
else:
results.append(match.group(1))
except Exception as _:
print("Problem found parsing {} with pattern \"{}\"".format(base_url, pattern))
results.append(None)
return results
new = 0
urls = parse_pages([d['base_url'] for d in data],
[d['pattern'] for d in data],
[d['combine_base'] if 'combine_base' in d.keys() else False for d in data])
for d, url in zip(data, urls):
print("Getting %s ..." % d['name'])
comic = Comic.query.filter_by(name=d['name']).first()
if url is not None:
print("Found %s ..." % url)
if comic is None:
comic = Comic()
comic.name = d['name']
comic.base_url = d['base_url']
comic.img_url = url
db.session.add(comic)
new += 1
else:
if comic.img_url != url:
comic.img_url = url
new += 1
db.session.commit()
msg = Message()
if new > 1:
msg.message = 'Update complete: found %d new comics' % new
elif new == 1:
msg.message = 'Update complete: found %d new comic' % new
else:
msg.message = 'Update complete: no new comics found'
db.session.add(msg)
db.session.commit()
return True