diff --git a/config.js b/config.js index ef3ed3f..3e1165c 100644 --- a/config.js +++ b/config.js @@ -6,6 +6,7 @@ dotenv.config({ path: 'data/config.env' }); // loads env vars from file - will n // Options - also see table in README.md export const cfg = { debug: process.env.PWDEBUG == '1', // runs non-headless and opens https://playwright.dev/docs/inspector + record: process.env.RECORD == '1', // `recordHar` (network) + `recordVideo` dryrun: process.env.DRYRUN == '1', // don't claim anything show: process.env.SHOW == '1', // run non-headless get headless() { return !this.debug && !this.show }, diff --git a/epic-games.js b/epic-games.js index a9e2c27..23c0eeb 100644 --- a/epic-games.js +++ b/epic-games.js @@ -40,7 +40,8 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, { userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 locale: "en-US", // ignore OS locale to be sure to have english text for locators - // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated + recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800 + recordHar: cfg.record ? { path: `data/record/eg-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools args: [ // https://peter.sh/experiments/chromium-command-line-switches // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' // '--restore-last-session', // does not apply for crash/killed @@ -58,6 +59,12 @@ if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist // console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); +if (cfg.record && cfg.debug) { + // const filter = _ => true; + const filter = r => r.url().includes('store.epicgames.com'); + page.on('request', request => filter(request) && console.log('>>', request.method(), request.url())); + page.on('response', response => filter(response) && console.log('<<', response.status(), response.url())); +} const notify_games = []; let user; @@ -182,6 +189,7 @@ try { if (cfg.debug) await page.pause(); if (cfg.dryrun) { console.log(' DRYRUN=1 -> Skip order!'); + notify_game.status = 'skipped'; continue; } @@ -232,7 +240,7 @@ try { notify(`epic-games failed: ${error.message.split('\n')[0]}`); } finally { await db.write(); // write out json db - if (notify_games.filter(g => g.status != 'existed' && g.status != 'requires base game').length) { // don't notify if all were already claimed + if (notify_games.filter(g => g.status == 'claimed' || g.status == 'failed').length) { // don't notify if all have status 'existed', 'manual', 'requires base game', 'unavailable-in-region', 'skipped' notify(`epic-games (${user}):
${html_game_list(notify_games)}`); } } diff --git a/migrate.js b/migrate.js new file mode 100644 index 0000000..41bbe13 --- /dev/null +++ b/migrate.js @@ -0,0 +1,34 @@ +import { existsSync } from 'fs'; +import { Low } from 'lowdb'; +import { JSONFile } from 'lowdb/node'; +import { datetime } from './util.js'; + +const datetime_UTCtoLocalTimezone = async file => { + if (!existsSync(file)) + return console.error('File does not exist:', file); + const db = new Low(new JSONFile(file)); + await db.read(); + db.data ||= {}; + console.log('Migrating', file); + for (const user in db.data) { + for (const game in db.data[user]) { + const time1 = db.data[user][game].time; + const time1s = time1.endsWith('Z') ? time1 : time1 + ' UTC'; + const time2 = datetime(new Date(time1s)); + console.log([game, time1, time2]); + db.data[user][game].time = time2; + } + } + // console.log(db.data); + await db.write(); // write out json db +}; + +const args = process.argv.slice(2); +if (args[0] == 'localtime') { + const files = args.slice(1); + console.log('Will convert UTC datetime to local timezone for', files); + files.forEach(datetime_UTCtoLocalTimezone); +} else { + console.log('Usage: node migrate.js '); + console.log(' node migrate.js localtime data/*.json'); +} diff --git a/unrealengine.js b/unrealengine.js new file mode 100644 index 0000000..25be32e --- /dev/null +++ b/unrealengine.js @@ -0,0 +1,195 @@ +// TODO This is mostly a copy of epic-games.js + +import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra +import { authenticator } from 'otplib'; +import path from 'path'; +import { existsSync, writeFileSync } from 'fs'; +import { jsonDb, datetime, stealth, filenamify, prompt, notify, html_game_list, handleSIGINT } from './util.js'; +import { cfg } from './config.js'; + +const URL_CLAIM = 'https://www.unrealengine.com/marketplace/en-US/assets?count=20&sortBy=effectiveDate&sortDir=DESC&start=0&tag=4910'; +const URL_LOGIN = 'https://www.epicgames.com/id/login?lang=en-US&noHostRedirect=true&redirectUrl=' + URL_CLAIM; + +console.log(datetime(), 'started checking unrealengine'); + +const db = await jsonDb('unrealengine.json'); +db.data ||= {}; + +handleSIGINT(); + +// https://playwright.dev/docs/auth#multi-factor-authentication +const context = await firefox.launchPersistentContext(cfg.dir.browser, { + headless: cfg.headless, + viewport: { width: cfg.width, height: cfg.height }, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? + // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 + locale: "en-US", // ignore OS locale to be sure to have english text for locators +}); + +await stealth(context); + +if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); + +const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist +// console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); + +const notify_games = []; +let user; + +try { + await context.addCookies([{name: 'OptanonAlertBoxClosed', value: new Date(Date.now() - 5*24*60*60*1000).toISOString(), domain: '.epicgames.com', path: '/'}]); // Accept cookies to get rid of banner to save space on screen. Set accept time to 5 days ago. + + await page.goto(URL_CLAIM, { waitUntil: 'domcontentloaded' }); // 'domcontentloaded' faster than default 'load' https://playwright.dev/docs/api/class-page#page-goto + + while (await page.locator('a[role="button"]:has-text("Sign In")').count() > 0) { + console.error('Not signed in anymore. Please login in the browser or here in the terminal.'); + if (cfg.novnc_port) console.info(`Open http://localhost:${cfg.novnc_port} to login inside the docker container.`); + if (!cfg.debug) context.setDefaultTimeout(cfg.login_timeout); // give user some extra time to log in + console.info(`Login timeout is ${cfg.login_timeout/1000} seconds!`); + await page.goto(URL_LOGIN, { waitUntil: 'domcontentloaded' }); + if (cfg.eg_email && cfg.eg_password) console.info('Using email and password from environment.'); + else console.info('Press ESC to skip the prompts if you want to login in the browser (not possible in headless mode).'); + const email = cfg.eg_email || await prompt({message: 'Enter email'}); + const password = email && (cfg.eg_password || await prompt({type: 'password', message: 'Enter password'})); + if (email && password) { + await page.click('text=Sign in with Epic Games'); + await page.fill('#email', email); + await page.fill('#password', password); + await page.click('button[type="submit"]'); + page.waitForSelector('#h_captcha_challenge_login_prod iframe').then(() => { + console.error('Got a captcha during login (likely due to too many attempts)! You may solve it in the browser, get a new IP or try again in a few hours.'); + notify('unrealengine: got captcha during login. Please check.'); + }).catch(_ => { }); + // handle MFA, but don't await it + page.waitForURL('**/id/login/mfa**').then(async () => { + console.log('Enter the security code to continue - This appears to be a new device, browser or location. A security code has been sent to your email address at ...'); + // TODO locator for text (email or app?) + const otp = cfg.eg_otpkey && authenticator.generate(cfg.eg_otpkey) || await prompt({type: 'text', message: 'Enter two-factor sign in code', validate: n => n.toString().length == 6 || 'The code must be 6 digits!'}); // can't use type: 'number' since it strips away leading zeros and codes sometimes have them + await page.type('input[name="code-input-0"]', otp.toString()); + await page.click('button[type="submit"]'); + }).catch(_ => { }); + } else { + console.log('Waiting for you to login in the browser.'); + await notify('unrealengine: no longer signed in and not enough options set for automatic login.'); + if (cfg.headless) { + console.log('Run `SHOW=1 node unrealengine` to login in the opened browser.'); + await context.close(); // finishes potential recording + process.exit(1); + } + } + await page.waitForURL(URL_CLAIM); + if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); + } + await page.waitForTimeout(1000); + user = await page.locator('.user-label').first().innerHTML(); + console.log(`Signed in as ${user}`); + db.data[user] ||= {}; + + page.locator('button:has-text("Accept All Cookies")').click().catch(_ => { }); + + const ids = []; + for (const p of await page.locator('article.asset').all()) { + const link = p.locator('h3 a'); + const title = await link.innerText(); + const url = 'https://www.unrealengine.com' + await link.getAttribute('href'); + console.log([title, url]); + const id = url.split('/').pop(); + db.data[user][id] ||= { title, time: datetime(), url, status: 'failed' }; // this will be set on the initial run only! + const notify_game = { title, url, status: 'failed' }; + notify_games.push(notify_game); // status is updated below + // if (await p.locator('.btn .add-review-btn').count()) { // did not work + if((await p.getAttribute('class')).includes('asset--owned')) { + console.log(' ↳ Already claimed'); + if (db.data[user][id].status != 'claimed') { + db.data[user][id].status = 'existed'; + notify_game.status = 'existed'; + } + continue; + } + if (await p.locator('.btn .in-cart').count()) { + console.log(' ↳ Already in cart'); + continue; + } + await p.locator('.btn .add').click(); + console.log(' ↳ Added to cart'); + ids.push(id); + } + if (!ids.length) { + console.log('Nothing to claim'); + } else { + const price = (await page.locator('.shopping-cart .total .price').innerText()).split(' '); + console.log('Price: ', price[1], 'instead of', price[0]); + if (price[1] != '0') { + console.error('Price is not 0! Exit!'); + process.exit(1); + } + // await page.pause(); + console.log('Click shopping cart'); + await page.locator('.shopping-cart').click(); + // await page.waitForTimeout(2000); + await page.locator('button.checkout').click(); + console.log('Click checkout'); + // maybe: Accept End User License Agreement + page.locator('[name=accept-label]').check().then(() => { + console.log('Accept End User License Agreement'); + page.locator('span:text-is("Accept")').click() // otherwise matches 'Accept All Cookies' + }).catch(_ => { }); + await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? + const iframe = page.frameLocator('#webPurchaseContainer iframe'); + + if (cfg.debug) await page.pause(); + if (cfg.dryrun) { + console.log('DRYRUN=1 -> Skip order!'); + throw new Error('DRYRUN=1'); + } + + console.log('Click Place Order'); + // Playwright clicked before button was ready to handle event, https://github.com/vogler/free-games-claimer/issues/84#issuecomment-1474346591 + await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 }); + + // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 + const btnAgree = iframe.locator('button:has-text("I Agree")'); + btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree' + try { + // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? + const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); + captcha.waitFor().then(async () => { // don't await, since element may not be shown + // console.info(' Got hcaptcha challenge! NopeCHA extension will likely solve it.') + console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') + }).catch(_ => { }); // may time out if not shown + await page.waitForSelector('text=Thank you'); + for (const id of ids) { + db.data[user][id].status = 'claimed'; + db.data[user][id].time = datetime(); // claimed time overwrites failed/dryrun time + } + notify_games.forEach(g => g.status == 'failed' && (g.status = 'claimed')); + console.log('Claimed successfully!'); + // context.setDefaultTimeout(cfg.timeout); + } catch (e) { + console.log(e); + // console.error(' Failed to claim! Try again if NopeCHA timed out. Click the extension to see if you ran out of credits (refill after 24h). To avoid captchas try to get a new IP or set a cookie from https://www.hcaptcha.com/accessibility'); + console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); + const p = path.resolve(cfg.dir.screenshots, 'unrealengine', 'failed', `${filenamify(datetime())}.png`); + await page.screenshot({ path: p, fullPage: true }); + // db.data[user][id].status = 'failed'; + notify_games.forEach(g => g.status = 'failed'); + } + // notify_game.status = db.data[user][game_id].status; // claimed or failed + + const p = path.resolve(cfg.dir.screenshots, 'unrealengine', `${filenamify(datetime())}.png`); + if (notify_games.length) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... + console.log('Done'); + } +} catch (error) { + console.error(error); // .toString()? + process.exitCode ||= 1; + if (error.message && process.exitCode != 130) + notify(`unrealengine failed: ${error.message.split('\n')[0]}`); +} finally { + await db.write(); // write out json db + if (notify_games.filter(g => g.status != 'existed').length) { // don't notify if all were already claimed + notify(`unrealengine (${user}):
${html_game_list(notify_games)}`); + } +} +if (cfg.debug) writeFileSync(path.resolve(cfg.dir.browser, 'cookies.json'), JSON.stringify(await context.cookies())); +await context.close(); diff --git a/util.js b/util.js index 839e5c8..2725745 100644 --- a/util.js +++ b/util.js @@ -19,9 +19,9 @@ export const jsonDb = async file => { export const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); // date and time as UTC (no timezone offset) in nicely readable and sortable format, e.g., 2022-10-06 12:05:27.313 -export const datetime = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', ''); -// same as datetime() but for local timezone, e.g., UTC + 2h for the above in DE -export const datetimeLocal = (d = new Date()) => datetime(new Date(d.getTime() - new Date().getTimezoneOffset() * 60000)); +export const datetimeUTC = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', ''); +// same as datetimeUTC() but for local timezone, e.g., UTC + 2h for the above in DE +export const datetime = (d = new Date()) => datetimeUTC(new Date(d.getTime() - d.getTimezoneOffset() * 60000)); export const filenamify = s => s.replaceAll(':', '.').replace(/[^a-z0-9 _\-.]/gi, '_'); // alternative: https://www.npmjs.com/package/filenamify - On Unix-like systems, / is reserved. On Windows, <>:"/\|?* along with trailing periods are reserved. export const handleSIGINT = () => process.on('SIGINT', () => { // e.g. when killed by Ctrl-C