Skip to content

Commit

Permalink
fix: timeout puppeter
Browse files Browse the repository at this point in the history
  • Loading branch information
@zcrossoverz
zcrossoverz committed Aug 2, 2023
1 parent b5fd272 commit ee30da7
Show file tree
Hide file tree
Showing 9 changed files with 34 additions and 6 deletions.
5 changes: 5 additions & 0 deletions dist/src/lib/asurascans.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class AsuraScans {
search(keyword, page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -68,6 +69,7 @@ class AsuraScans {
getListByGenre(genre, page, status, sort) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
const url = `${this.baseUrl}${genre.path}${page !== undefined && page > 1 ? `/page/${page}` : ``}`;
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
Expand Down Expand Up @@ -109,6 +111,7 @@ class AsuraScans {
getDataChapter(url_chapter, url, path, prev_chapter, next_chapter) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -151,6 +154,7 @@ class AsuraScans {
getDetailManga(url) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -216,6 +220,7 @@ class AsuraScans {
getListLatestUpdate(page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down
5 changes: 5 additions & 0 deletions dist/src/lib/blogtruyen.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class Blogtruyen {
search(keyword, page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -60,6 +61,7 @@ class Blogtruyen {
return __awaiter(this, void 0, void 0, function* () {
const path = genre.path;
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -96,6 +98,7 @@ class Blogtruyen {
url = url !== undefined ? url : '';
path = path !== undefined ? path : '';
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.goto(url_chapter);
const content = yield _page.$('#readonline > section');
const title = (0, validate_1.not_null)(yield _page.$eval('#readonline > header > h1', (el) => el.textContent));
Expand Down Expand Up @@ -148,6 +151,7 @@ class Blogtruyen {
getDetailManga(url) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -213,6 +217,7 @@ class Blogtruyen {
getListLatestUpdate(page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down
5 changes: 5 additions & 0 deletions dist/src/lib/nettruyen.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class Nettruyen {
search(keyword, page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -76,6 +77,7 @@ class Nettruyen {
getListByGenre(genre, page, status, sort) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
let path = genre.path;
if (sort !== undefined) {
path += `?sort=${sort}${status !== undefined ? `&status=${status}` : '&status=-1'}${page !== undefined ? `&page=${page}` : ''}`;
Expand Down Expand Up @@ -129,6 +131,7 @@ class Nettruyen {
url = url !== undefined ? url : '';
path = path !== undefined ? path : '';
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -200,6 +203,7 @@ class Nettruyen {
getDetailManga(url) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down Expand Up @@ -271,6 +275,7 @@ class Nettruyen {
getListLatestUpdate(page = 1) {
return __awaiter(this, void 0, void 0, function* () {
const _page = yield (yield this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
yield _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document')
Expand Down
2 changes: 1 addition & 1 deletion dist/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
const src_1 = require("./src");
const t = new src_1.Manga().build(src_1.MangaType.ASURASCANS);
void (() => __awaiter(void 0, void 0, void 0, function* () {
console.log(yield t.getListByGenre({ path: '/genres/action', name: 'action' }));
console.log(yield t.getListLatestUpdate());
}))();
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "manga-lib",
"author": "Nhan Nguyen",
"version": "1.1.1",
"version": "1.1.2",
"license": "MIT",
"description": "A library for scraping manga from various websites.",
"repository": "https://github.com/zcrossoverz/manga-lib",
Expand Down
5 changes: 5 additions & 0 deletions src/lib/asurascans.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export class AsuraScans implements AbstractMangaFactory {

async search(keyword: string, page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -91,6 +92,7 @@ export class AsuraScans implements AbstractMangaFactory {
sort?: any
): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
const url = `${this.baseUrl}${genre.path}${
page !== undefined && page > 1 ? `/page/${page}` : ``
}`;
Expand Down Expand Up @@ -150,6 +152,7 @@ export class AsuraScans implements AbstractMangaFactory {
next_chapter?: chapter
): Promise<responseChapter> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -203,6 +206,7 @@ export class AsuraScans implements AbstractMangaFactory {

async getDetailManga(url: string): Promise<responseDetailManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -307,6 +311,7 @@ export class AsuraScans implements AbstractMangaFactory {

async getListLatestUpdate(page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down
7 changes: 6 additions & 1 deletion src/lib/blogtruyen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export class Blogtruyen implements AbstractMangaFactory {

async search(keyword: string, page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -90,6 +91,7 @@ export class Blogtruyen implements AbstractMangaFactory {
): Promise<responseListManga> {
const path = genre.path;
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -148,6 +150,7 @@ export class Blogtruyen implements AbstractMangaFactory {
path = path !== undefined ? path : '';

const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.goto(url_chapter);
const content = await _page.$('#readonline > section');
const title = not_null(
Expand Down Expand Up @@ -211,6 +214,7 @@ export class Blogtruyen implements AbstractMangaFactory {

async getDetailManga(url: string): Promise<responseDetailManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -304,12 +308,13 @@ export class Blogtruyen implements AbstractMangaFactory {

async getListLatestUpdate(page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
else req.continue();
});
await _page.goto(`${this.baseUrl}${page > 1 ? `/page-${page}` :``}`);
await _page.goto(`${this.baseUrl}${page > 1 ? `/page-${page}` : ``}`);
const element = await _page.$$(
'#wrapper > section.main-content > div > div:nth-child(1) > section.list-mainpage > div:nth-child(1) > div > div'
);
Expand Down
5 changes: 5 additions & 0 deletions src/lib/nettruyen.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ export class Nettruyen implements AbstractMangaFactory {

async search(keyword: string, page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -120,6 +121,7 @@ export class Nettruyen implements AbstractMangaFactory {
sort?: NETTRUYEN_SORT_FILTER
): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
let path = genre.path;
if (sort !== undefined) {
path += `?sort=${sort}${
Expand Down Expand Up @@ -208,6 +210,7 @@ export class Nettruyen implements AbstractMangaFactory {
path = path !== undefined ? path : '';

const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -301,6 +304,7 @@ export class Nettruyen implements AbstractMangaFactory {

async getDetailManga(url: string): Promise<responseDetailManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down Expand Up @@ -419,6 +423,7 @@ export class Nettruyen implements AbstractMangaFactory {

async getListLatestUpdate(page = 1): Promise<responseListManga> {
const _page = await (await this.browser).newPage();
_page.setDefaultNavigationTimeout(0);
await _page.setRequestInterception(true);
_page.on('request', (req) => {
if (req.resourceType() !== 'document') req.abort();
Expand Down
4 changes: 1 addition & 3 deletions test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,5 @@ import { Manga, MangaType } from './src';
const t = new Manga().build(MangaType.ASURASCANS);

void (async () => {
console.log(
await t.getListByGenre({ path: '/genres/action', name: 'action' })
);
console.log(await t.getListLatestUpdate());
})();

0 comments on commit ee30da7

Please sign in to comment.