tappytoon: unbreak
This commit is contained in:
Alexandra Dunn
parent
8c8c4b6ed9
commit
aab0e045f5
|
|
@ -1,39 +0,0 @@
|
|||
const fs = require('fs')
|
||||
const puppeteer = require('puppeteer')
|
||||
|
||||
const LOGIN_PAGE = 'https://www.tappytoon.com/login'
|
||||
const USER = process.env.TAPPY_USER
|
||||
const PASS = process.env.TAPPY_PASS
|
||||
|
||||
if (!USER) {
|
||||
console.log('--- ERROR: env variable TAPPY_USER unset.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (!PASS) {
|
||||
console.log('--- ERROR: env variable TAPPY_PASS unset.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
puppeteer.launch().then(async browser => {
|
||||
const page = await browser.newPage()
|
||||
await page.goto(LOGIN_PAGE)
|
||||
|
||||
const userField = await page.$('input[type=email]')
|
||||
const passField = await page.$('input[type=password]')
|
||||
|
||||
await userField.type(USER)
|
||||
await passField.type(PASS)
|
||||
|
||||
await page.click('.MuiButtonBase-root')
|
||||
await page.waitForNavigation()
|
||||
|
||||
const cookies = await page.cookies()
|
||||
|
||||
// I'm not sure why puppeteer doesn't parse tappytoon's cookies correctly
|
||||
// fs.writeFileSync('cookies.json', decodeURIComponent(JSON.stringify(cookies).replace(/%3D%3D/, '=')))
|
||||
fs.writeFileSync('cookies.json', JSON.stringify(cookies))
|
||||
|
||||
await page.close()
|
||||
await browser.close()
|
||||
})
|
||||
|
|
@ -1,103 +1,55 @@
|
|||
const fs = require('fs')
|
||||
const got = require('got')
|
||||
const util = require('util')
|
||||
const puppeteer = require('puppeteer')
|
||||
(async function () {
|
||||
const fs = require('fs')
|
||||
const got = require('got')
|
||||
const util = require('util')
|
||||
|
||||
const COMIC_PAGE_BASE = 'https://www.tappytoon.com/comics/'
|
||||
const COMIC_SHORT_NAME = process.env.TAPPY_COMIC
|
||||
const COMIC = `${COMIC_PAGE_BASE}${COMIC_SHORT_NAME}`
|
||||
const COMIC_NO = process.env.TAPPY_EP
|
||||
const COOKIES = JSON.parse(fs.readFileSync('cookies.json'))
|
||||
const {promisify} = require('util');
|
||||
const stream = require('stream');
|
||||
const pipeline = promisify(stream.pipeline)
|
||||
|
||||
// const LOGIN_PAGE = 'https://www.tappytoon.com/login'
|
||||
// const USER = process.env.TAPPY_USER
|
||||
// const PASS = process.env.TAPPY_PASS
|
||||
const SKIP = parseInt(process.env.SKIP) || 0
|
||||
const COMIC_ID = process.env.COMIC_ID
|
||||
|
||||
// if (!USER) {
|
||||
// console.log('--- ERROR: env variable TAPPY_USER unset.')
|
||||
// process.exit(1)
|
||||
// }
|
||||
const TOKEN = process.env.TOKEN
|
||||
|
||||
// if (!PASS) {
|
||||
// console.log('--- ERROR: env variable TAPPY_PASS unset.')
|
||||
// process.exit(1)
|
||||
// }
|
||||
const API_BASE = 'https://api-global.tappytoon.com'
|
||||
|
||||
if (!COMIC_SHORT_NAME) {
|
||||
console.log()
|
||||
console.log('You must specify a comic with the TAPPY_COMIC environment variable.')
|
||||
console.log()
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
puppeteer.launch({ headless: true }).then(async browser => {
|
||||
const page = await browser.newPage()
|
||||
|
||||
for (const cookie of COOKIES) {
|
||||
// console.dir(cookie)
|
||||
await page.setCookie(cookie)
|
||||
}
|
||||
|
||||
await page.goto(COMIC)
|
||||
|
||||
let episodes = await page.evaluate(() => {
|
||||
return Promise.resolve(window.__NEXT_DATA__.props.initialProps.initialState.entities.chapters)
|
||||
})
|
||||
|
||||
let chapters = []
|
||||
for (let key in episodes) {
|
||||
chapters.push(episodes[key])
|
||||
}
|
||||
|
||||
if (COMIC_NO) {
|
||||
chapters = chapters.filter(e => {
|
||||
return COMIC_NO === e.title
|
||||
})
|
||||
}
|
||||
|
||||
// console.dir(chapters)
|
||||
|
||||
for (const ep of chapters) {
|
||||
console.log(`-- Downloading "${ep.title}"`)
|
||||
|
||||
await page.goto(`https://www.tappytoon.com/chapters/${ep.id}`)
|
||||
|
||||
try {
|
||||
await page.waitFor(() => {
|
||||
return Promise.resolve(document.getElementsByTagName('img').length > 1)
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(`--- ERROR: failed to load https://www.tappytoon.com/chapters/${ep.id}`)
|
||||
continue
|
||||
let product_url = `${API_BASE}/comics/${COMIC_ID}/chapters?excludes=wait_until_free&sort=asc&locale=en`
|
||||
const options = {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Authorization': `Bearer ${TOKEN}`,
|
||||
}
|
||||
}
|
||||
|
||||
let panels = await page.$$eval('img', images => images.map(img => img.src))
|
||||
let response = await got(product_url, options)
|
||||
let product = JSON.parse(response.body)
|
||||
|
||||
panels = panels.filter(e => {
|
||||
return e !== 'https://static.tappytoon.com/assets/images/copyright-warning.png'
|
||||
})
|
||||
// console.dir(panels)
|
||||
|
||||
let destination = `downloads/${COMIC_SHORT_NAME}/${ep.order}`
|
||||
for (const ep of product.slice(SKIP)) {
|
||||
let destination = `downloads/${ep.comicId}/${ep.order.toString().padStart(3, '0')}`
|
||||
console.log(`Creating directory ${destination}`)
|
||||
fs.mkdirSync(destination, { recursive: true })
|
||||
|
||||
let n = panels.length
|
||||
while (n--) {
|
||||
try {
|
||||
let filename = `${destination}/${n}.jpg`
|
||||
console.log(`-- Downloading ${panels[n]}`)
|
||||
got.stream(panels[n]).pipe(fs.createWriteStream(filename))
|
||||
let episode_url = `${API_BASE}/content-delivery/contents?chapterId=${ep.id}&variant=high&locale=en`
|
||||
console.log(episode_url)
|
||||
|
||||
// don't try to download everything all at once
|
||||
await new Promise(s => setTimeout(s, 500))
|
||||
} catch (err) {
|
||||
console.error(`--- ERROR: ${err}`)
|
||||
let response = await got(episode_url, options)
|
||||
let blob = JSON.parse(response.body)
|
||||
|
||||
for (const page of blob.media) {
|
||||
let paddedN = `${page.sortKey}`.padStart(3, '0')
|
||||
let filename = `${destination}/${paddedN}.jpg`
|
||||
|
||||
if (await fs.existsSync(filename)) {
|
||||
console.log(`-- ${filename} already downloaded, skipping ...`)
|
||||
} else {
|
||||
console.log(page.url)
|
||||
await pipeline(
|
||||
got.stream(`${page.url}`),
|
||||
fs.createWriteStream(filename)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await page.close()
|
||||
await browser.close()
|
||||
}).catch(err => {
|
||||
return console.log(util.inspect(err))
|
||||
})
|
||||
})()
|
||||
|
|
|
|||
|
|
@ -1,27 +1,12 @@
|
|||
# tappytoon
|
||||
|
||||
the script authenticates with lezhin using a pre-generated cookies file.
|
||||
currently puppeteer mangles the cookies file, so you have to fetch the cookies
|
||||
another way (e.g. a browser plugin) and replace the values in the generated
|
||||
`cookies.json`.
|
||||
use network inspector to find the comic id and your auth token from the comic page:
|
||||
|
||||
```
|
||||
TAPPY_USER=<twitter username> TAPPY_PASS=<twitter password> node auth/local.js
|
||||
[SKIP=N] COMIC_ID=92 TOKEN=<long string> node download.js
|
||||
```
|
||||
|
||||
this will write to `cookies.json`, which is read by `download.js`.
|
||||
image files will be written to e.g. `downloads/92`.
|
||||
|
||||
specify the comic to download using the comic’s slug, which is what appears in
|
||||
the URL (e.g., `https://www.tappytoon.com/comics/hershimch`):
|
||||
|
||||
```
|
||||
TAPPY_COMIC=hershimch node download.js
|
||||
```
|
||||
|
||||
image files will be written to `downloads/hershimch`.
|
||||
|
||||
individual chapters can be downloaded with the `TAPPY_EP` variable:
|
||||
|
||||
```
|
||||
TAPPY_EP='Episode 22' TAPPY_COMIC=hershimch node download.js
|
||||
```
|
||||
|
|
|
|||
Loading…
Reference in a new issue