From bb51fd80657fafcf974256024c9000ce501d2815 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 28 Apr 2023 00:11:29 +0200 Subject: [PATCH 1/8] use local time instead of UTC, migrate.js, closes #131 Run `node migrate.js localtime data/*.json` to convert existing `time` entries from UTC to your local timezone. --- migrate.js | 34 ++++++++++++++++++++++++++++++++++ util.js | 6 +++--- 2 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 migrate.js diff --git a/migrate.js b/migrate.js new file mode 100644 index 0000000..41bbe13 --- /dev/null +++ b/migrate.js @@ -0,0 +1,34 @@ +import { existsSync } from 'fs'; +import { Low } from 'lowdb'; +import { JSONFile } from 'lowdb/node'; +import { datetime } from './util.js'; + +const datetime_UTCtoLocalTimezone = async file => { + if (!existsSync(file)) + return console.error('File does not exist:', file); + const db = new Low(new JSONFile(file)); + await db.read(); + db.data ||= {}; + console.log('Migrating', file); + for (const user in db.data) { + for (const game in db.data[user]) { + const time1 = db.data[user][game].time; + const time1s = time1.endsWith('Z') ? time1 : time1 + ' UTC'; + const time2 = datetime(new Date(time1s)); + console.log([game, time1, time2]); + db.data[user][game].time = time2; + } + } + // console.log(db.data); + await db.write(); // write out json db +}; + +const args = process.argv.slice(2); +if (args[0] == 'localtime') { + const files = args.slice(1); + console.log('Will convert UTC datetime to local timezone for', files); + files.forEach(datetime_UTCtoLocalTimezone); +} else { + console.log('Usage: node migrate.js '); + console.log(' node migrate.js localtime data/*.json'); +} diff --git a/util.js b/util.js index 839e5c8..2725745 100644 --- a/util.js +++ b/util.js @@ -19,9 +19,9 @@ export const jsonDb = async file => { export const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); // date and time as UTC (no timezone offset) in nicely readable and sortable format, e.g., 2022-10-06 12:05:27.313 -export const datetime = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', ''); -// same as datetime() but for local timezone, e.g., UTC + 2h for the above in DE -export const datetimeLocal = (d = new Date()) => datetime(new Date(d.getTime() - new Date().getTimezoneOffset() * 60000)); +export const datetimeUTC = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', ''); +// same as datetimeUTC() but for local timezone, e.g., UTC + 2h for the above in DE +export const datetime = (d = new Date()) => datetimeUTC(new Date(d.getTime() - d.getTimezoneOffset() * 60000)); export const filenamify = s => s.replaceAll(':', '.').replace(/[^a-z0-9 _\-.]/gi, '_'); // alternative: https://www.npmjs.com/package/filenamify - On Unix-like systems, / is reserved. On Windows, <>:"/\|?* along with trailing periods are reserved. export const handleSIGINT = () => process.on('SIGINT', () => { // e.g. when killed by Ctrl-C From 5214bea488ae86824713d28e25c57d769fa8fdab Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 28 Apr 2023 00:26:09 +0200 Subject: [PATCH 2/8] cp epic-games.js unrealengine.js --- unrealengine.js | 232 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 unrealengine.js diff --git a/unrealengine.js b/unrealengine.js new file mode 100644 index 0000000..3819083 --- /dev/null +++ b/unrealengine.js @@ -0,0 +1,232 @@ +import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra +import { authenticator } from 'otplib'; +import path from 'path'; +import { existsSync, writeFileSync } from 'fs'; +import { jsonDb, datetime, stealth, filenamify, prompt, notify, html_game_list, handleSIGINT } from './util.js'; +import { cfg } from './config.js'; + +const URL_CLAIM = 'https://store.epicgames.com/en-US/free-games'; +const URL_LOGIN = 'https://www.epicgames.com/id/login?lang=en-US&noHostRedirect=true&redirectUrl=' + URL_CLAIM; + +console.log(datetime(), 'started checking epic-games'); + +const db = await jsonDb('epic-games.json'); +db.data ||= {}; + +handleSIGINT(); + +// https://www.nopecha.com extension source from https://github.com/NopeCHA/NopeCHA/releases/tag/0.1.16 +// const ext = path.resolve('nopecha'); // used in Chromium, currently not needed in Firefox + +// https://playwright.dev/docs/auth#multi-factor-authentication +const context = await firefox.launchPersistentContext(cfg.dir.browser, { + // chrome will not work in linux arm64, only chromium + // channel: 'chrome', // https://playwright.dev/docs/browsers#google-chrome--microsoft-edge + headless: cfg.headless, + viewport: { width: cfg.width, height: cfg.height }, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? + // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 + locale: "en-US", // ignore OS locale to be sure to have english text for locators + // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated + args: [ // https://peter.sh/experiments/chromium-command-line-switches + // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' + // '--restore-last-session', // does not apply for crash/killed + '--hide-crash-restore-bubble', + // `--disable-extensions-except=${ext}`, + // `--load-extension=${ext}`, + ], + // ignoreDefaultArgs: ['--enable-automation'], // remove default arg that shows the info bar with 'Chrome is being controlled by automated test software.'. Since Chromeium 106 this leads to show another info bar with 'You are using an unsupported command-line flag: --no-sandbox. Stability and security will suffer.'. +}); + +// Without stealth plugin, the website shows an hcaptcha on login with username/password and in the last step of claiming a game. It may have other heuristics like unsuccessful logins as well. After <6h (TBD) it resets to no captcha again. Getting a new IP also resets. +await stealth(context); + +if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); + +const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist +// console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); + +const notify_games = []; +let user; + +try { + await context.addCookies([{name: 'OptanonAlertBoxClosed', value: new Date(Date.now() - 5*24*60*60*1000).toISOString(), domain: '.epicgames.com', path: '/'}]); // Accept cookies to get rid of banner to save space on screen. Set accept time to 5 days ago. + + await page.goto(URL_CLAIM, { waitUntil: 'domcontentloaded' }); // 'domcontentloaded' faster than default 'load' https://playwright.dev/docs/api/class-page#page-goto + + // page.click('button:has-text("Accept All Cookies")').catch(_ => { }); // Not needed anymore since we set the cookie above. Clicking this did not always work since the message was animated in too slowly. + + while (await page.locator('a[role="button"]:has-text("Sign In")').count() > 0) { + console.error('Not signed in anymore. Please login in the browser or here in the terminal.'); + if (cfg.novnc_port) console.info(`Open http://localhost:${cfg.novnc_port} to login inside the docker container.`); + if (!cfg.debug) context.setDefaultTimeout(cfg.login_timeout); // give user some extra time to log in + console.info(`Login timeout is ${cfg.login_timeout/1000} seconds!`); + await page.goto(URL_LOGIN, { waitUntil: 'domcontentloaded' }); + if (cfg.eg_email && cfg.eg_password) console.info('Using email and password from environment.'); + else console.info('Press ESC to skip the prompts if you want to login in the browser (not possible in headless mode).'); + const email = cfg.eg_email || await prompt({message: 'Enter email'}); + const password = email && (cfg.eg_password || await prompt({type: 'password', message: 'Enter password'})); + if (email && password) { + await page.click('text=Sign in with Epic Games'); + await page.fill('#email', email); + await page.fill('#password', password); + await page.click('button[type="submit"]'); + page.waitForSelector('#h_captcha_challenge_login_prod iframe').then(() => { + console.error('Got a captcha during login (likely due to too many attempts)! You may solve it in the browser, get a new IP or try again in a few hours.'); + notify('epic-games: got captcha during login. Please check.'); + }).catch(_ => { }); + // handle MFA, but don't await it + page.waitForURL('**/id/login/mfa**').then(async () => { + console.log('Enter the security code to continue - This appears to be a new device, browser or location. A security code has been sent to your email address at ...'); + // TODO locator for text (email or app?) + const otp = cfg.eg_otpkey && authenticator.generate(cfg.eg_otpkey) || await prompt({type: 'text', message: 'Enter two-factor sign in code', validate: n => n.toString().length == 6 || 'The code must be 6 digits!'}); // can't use type: 'number' since it strips away leading zeros and codes sometimes have them + await page.type('input[name="code-input-0"]', otp.toString()); + await page.click('button[type="submit"]'); + }).catch(_ => { }); + } else { + console.log('Waiting for you to login in the browser.'); + await notify('epic-games: no longer signed in and not enough options set for automatic login.'); + if (cfg.headless) { + console.log('Run `SHOW=1 node epic-games` to login in the opened browser.'); + await context.close(); // finishes potential recording + process.exit(1); + } + } + await page.waitForURL(URL_CLAIM); + if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); + } + user = await page.locator('#user span').first().innerHTML(); + console.log(`Signed in as ${user}`); + db.data[user] ||= {}; + + // Detect free games + const game_loc = page.locator('a:has(span:text-is("Free Now"))'); + await game_loc.last().waitFor(); + // clicking on `game_sel` sometimes led to a 404, see https://github.com/vogler/free-games-claimer/issues/25 + // debug showed that in those cases the href was still correct, so we `goto` the urls instead of clicking. + // Alternative: parse the json loaded to build the page https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions + // filter data.Catalog.searchStore.elements for .promotions.promotionalOffers being set and build URL with .catalogNs.mappings[0].pageSlug or .urlSlug if not set to some wrong id like it was the case for spirit-of-the-north-f58a66 - this is also what's done here: https://github.com/claabs/epicgames-freegames-node/blob/938a9653ffd08b8284ea32cf01ac8727d25c5d4c/src/puppet/free-games.ts#L138-L213 + const urlSlugs = await Promise.all((await game_loc.elementHandles()).map(a => a.getAttribute('href'))); + const urls = urlSlugs.map(s => 'https://store.epicgames.com' + s); + console.log('Free games:', urls); + + for (const url of urls) { + await page.goto(url); // , { waitUntil: 'domcontentloaded' }); + const btnText = await page.locator('//button[@data-testid="purchase-cta-button"][not(contains(.,"Loading"))]').first().innerText(); // barrier to block until page is loaded + + // click Continue if 'This game contains mature content recommended only for ages 18+' + if (await page.locator('button:has-text("Continue")').count() > 0) { + console.log(' This game contains mature content recommended only for ages 18+'); + await page.click('button:has-text("Continue")', { delay: 111 }); + await page.waitForTimeout(2000); + } + + const title = await page.locator('h1').first().innerText(); + const game_id = page.url().split('/').pop(); + db.data[user][game_id] ||= { title, time: datetime(), url: page.url() }; // this will be set on the initial run only! + console.log('Current free game:', title); + const notify_game = { title, url, status: 'failed' }; + notify_games.push(notify_game); // status is updated below + + if (btnText.toLowerCase() == 'in library') { + console.log(' Already in library! Nothing to claim.'); + notify_game.status = 'existed'; + db.data[user][game_id].status ||= 'existed'; // does not overwrite claimed or failed + if (db.data[user][game_id].status.startsWith('failed')) db.data[user][game_id].status = 'manual'; // was failed but now it's claimed + } else if (btnText.toLowerCase() == 'requires base game') { + console.log(' Requires base game! Nothing to claim.'); + notify_game.status = 'requires base game'; + db.data[user][game_id].status ||= 'failed:requires-base-game'; + // TODO claim base game if it is free + const baseUrl = 'https://store.epicgames.com' + await page.locator('a:has-text("Overview")').getAttribute('href'); + console.log(' Base game:', baseUrl); + // await page.click('a:has-text("Overview")'); + } else { // GET + console.log(' Not in library yet! Click GET.'); + await page.click('[data-testid="purchase-cta-button"]', { delay: 11 }); // got stuck here without delay (or mouse move), see #75, 1ms was also enough + + // click Continue if 'Device not supported. This product is not compatible with your current device.' - avoided by Windows userAgent? + page.click('button:has-text("Continue")').catch(_ => { }); // needed since change from Chromium to Firefox? + + // Accept End User License Agreement (only needed once) + page.locator('input#agree').waitFor().then(async () => { + console.log('Accept End User License Agreement (only needed once)'); + await page.locator('input#agree').check(); + await page.locator('button:has-text("Accept")').click(); + }).catch(_ => { }); + + // it then creates an iframe for the purchase + await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? + const iframe = page.frameLocator('#webPurchaseContainer iframe'); + // skip game if unavailable in region, https://github.com/vogler/free-games-claimer/issues/46 TODO check games for account's region + if (await iframe.locator(':has-text("unavailable in your region")').count() > 0) { + console.error(' This product is unavailable in your region!'); + db.data[user][game_id].status = notify_game.status = 'unavailable-in-region'; + continue; + } + + iframe.locator('.payment-pin-code').waitFor().then(async () => { + if (!cfg.eg_parentalpin) { + console.error(' EG_PARENTALPIN not set. Need to enter Parental Control PIN manually.'); + notify('epic-games: EG_PARENTALPIN not set. Need to enter Parental Control PIN manually.'); + } + await iframe.locator('input.payment-pin-code__input').first().type(cfg.eg_parentalpin); + await iframe.locator('button:has-text("Continue")').click({ delay: 11 }); + }).catch(_ => { }); + + if (cfg.debug) await page.pause(); + if (cfg.dryrun) { + console.log(' DRYRUN=1 -> Skip order!'); + continue; + } + + // Playwright clicked before button was ready to handle event, https://github.com/vogler/free-games-claimer/issues/84#issuecomment-1474346591 + await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 }); + + // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 + const btnAgree = iframe.locator('button:has-text("I Agree")'); + btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree' + try { + // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? + const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); + captcha.waitFor().then(async () => { // don't await, since element may not be shown + // console.info(' Got hcaptcha challenge! NopeCHA extension will likely solve it.') + console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') + // await page.waitForTimeout(2000); + // const p = path.resolve(cfg.dir.screenshots, 'epic-games', 'captcha', `${filenamify(datetime())}.png`); + // await captcha.screenshot({ path: p }); + // console.info(' Saved a screenshot of hcaptcha challenge to', p); + // console.error(' Got hcaptcha challenge. To avoid it, get a link from https://www.hcaptcha.com/accessibility'); // TODO save this link in config and visit it daily to set accessibility cookie to avoid captcha challenge? + }).catch(_ => { }); // may time out if not shown + await page.waitForSelector('text=Thanks for your order!'); + db.data[user][game_id].status = 'claimed'; + db.data[user][game_id].time = datetime(); // claimed time overwrites failed/dryrun time + console.log(' Claimed successfully!'); + // context.setDefaultTimeout(cfg.timeout); + } catch (e) { + console.log(e); + // console.error(' Failed to claim! Try again if NopeCHA timed out. Click the extension to see if you ran out of credits (refill after 24h). To avoid captchas try to get a new IP or set a cookie from https://www.hcaptcha.com/accessibility'); + console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); + const p = path.resolve(cfg.dir.screenshots, 'epic-games', 'failed', `${game_id}_${filenamify(datetime())}.png`); + await page.screenshot({ path: p, fullPage: true }); + db.data[user][game_id].status = 'failed'; + } + notify_game.status = db.data[user][game_id].status; // claimed or failed + + const p = path.resolve(cfg.dir.screenshots, 'epic-games', `${game_id}.png`); + if (!existsSync(p)) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... + } + } +} catch (error) { + console.error(error); // .toString()? + process.exitCode ||= 1; + if (error.message && process.exitCode != 130) + notify(`epic-games failed: ${error.message.split('\n')[0]}`); +} finally { + await db.write(); // write out json db + if (notify_games.filter(g => g.status != 'existed' && g.status != 'requires base game').length) { // don't notify if all were already claimed + notify(`epic-games (${user}):
${html_game_list(notify_games)}`); + } +} +if (cfg.debug) writeFileSync(path.resolve(cfg.dir.browser, 'cookies.json'), JSON.stringify(await context.cookies())); +await context.close(); \ No newline at end of file From 5809b0963aad0ce45aed04a2fe3377b3717c64d1 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 28 Apr 2023 00:59:27 +0200 Subject: [PATCH 3/8] ue: unrealengine: add assets to cart & checkout, rest same as for epic-games, #44 --- unrealengine.js | 224 ++++++++++++++++++------------------------------ 1 file changed, 82 insertions(+), 142 deletions(-) diff --git a/unrealengine.js b/unrealengine.js index 3819083..b7b7c63 100644 --- a/unrealengine.js +++ b/unrealengine.js @@ -1,3 +1,5 @@ +// TODO This is mostly a copy of epic-games.js + import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra import { authenticator } from 'otplib'; import path from 'path'; @@ -5,40 +7,25 @@ import { existsSync, writeFileSync } from 'fs'; import { jsonDb, datetime, stealth, filenamify, prompt, notify, html_game_list, handleSIGINT } from './util.js'; import { cfg } from './config.js'; -const URL_CLAIM = 'https://store.epicgames.com/en-US/free-games'; +const URL_CLAIM = 'https://www.unrealengine.com/marketplace/en-US/assets?count=20&sortBy=effectiveDate&sortDir=DESC&start=0&tag=4910'; const URL_LOGIN = 'https://www.epicgames.com/id/login?lang=en-US&noHostRedirect=true&redirectUrl=' + URL_CLAIM; -console.log(datetime(), 'started checking epic-games'); +console.log(datetime(), 'started checking unrealengine'); -const db = await jsonDb('epic-games.json'); +const db = await jsonDb('unrealengine.json'); db.data ||= {}; handleSIGINT(); -// https://www.nopecha.com extension source from https://github.com/NopeCHA/NopeCHA/releases/tag/0.1.16 -// const ext = path.resolve('nopecha'); // used in Chromium, currently not needed in Firefox - // https://playwright.dev/docs/auth#multi-factor-authentication const context = await firefox.launchPersistentContext(cfg.dir.browser, { - // chrome will not work in linux arm64, only chromium - // channel: 'chrome', // https://playwright.dev/docs/browsers#google-chrome--microsoft-edge headless: cfg.headless, viewport: { width: cfg.width, height: cfg.height }, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 locale: "en-US", // ignore OS locale to be sure to have english text for locators - // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated - args: [ // https://peter.sh/experiments/chromium-command-line-switches - // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' - // '--restore-last-session', // does not apply for crash/killed - '--hide-crash-restore-bubble', - // `--disable-extensions-except=${ext}`, - // `--load-extension=${ext}`, - ], - // ignoreDefaultArgs: ['--enable-automation'], // remove default arg that shows the info bar with 'Chrome is being controlled by automated test software.'. Since Chromeium 106 this leads to show another info bar with 'You are using an unsupported command-line flag: --no-sandbox. Stability and security will suffer.'. }); -// Without stealth plugin, the website shows an hcaptcha on login with username/password and in the last step of claiming a game. It may have other heuristics like unsuccessful logins as well. After <6h (TBD) it resets to no captcha again. Getting a new IP also resets. await stealth(context); if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); @@ -54,8 +41,6 @@ try { await page.goto(URL_CLAIM, { waitUntil: 'domcontentloaded' }); // 'domcontentloaded' faster than default 'load' https://playwright.dev/docs/api/class-page#page-goto - // page.click('button:has-text("Accept All Cookies")').catch(_ => { }); // Not needed anymore since we set the cookie above. Clicking this did not always work since the message was animated in too slowly. - while (await page.locator('a[role="button"]:has-text("Sign In")').count() > 0) { console.error('Not signed in anymore. Please login in the browser or here in the terminal.'); if (cfg.novnc_port) console.info(`Open http://localhost:${cfg.novnc_port} to login inside the docker container.`); @@ -73,7 +58,7 @@ try { await page.click('button[type="submit"]'); page.waitForSelector('#h_captcha_challenge_login_prod iframe').then(() => { console.error('Got a captcha during login (likely due to too many attempts)! You may solve it in the browser, get a new IP or try again in a few hours.'); - notify('epic-games: got captcha during login. Please check.'); + notify('unrealengine: got captcha during login. Please check.'); }).catch(_ => { }); // handle MFA, but don't await it page.waitForURL('**/id/login/mfa**').then(async () => { @@ -85,9 +70,9 @@ try { }).catch(_ => { }); } else { console.log('Waiting for you to login in the browser.'); - await notify('epic-games: no longer signed in and not enough options set for automatic login.'); + await notify('unrealengine: no longer signed in and not enough options set for automatic login.'); if (cfg.headless) { - console.log('Run `SHOW=1 node epic-games` to login in the opened browser.'); + console.log('Run `SHOW=1 node unrealengine` to login in the opened browser.'); await context.close(); // finishes potential recording process.exit(1); } @@ -95,138 +80,93 @@ try { await page.waitForURL(URL_CLAIM); if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); } - user = await page.locator('#user span').first().innerHTML(); + await page.waitForTimeout(1000); + user = await page.locator('.user-label').first().innerHTML(); console.log(`Signed in as ${user}`); db.data[user] ||= {}; - // Detect free games - const game_loc = page.locator('a:has(span:text-is("Free Now"))'); - await game_loc.last().waitFor(); - // clicking on `game_sel` sometimes led to a 404, see https://github.com/vogler/free-games-claimer/issues/25 - // debug showed that in those cases the href was still correct, so we `goto` the urls instead of clicking. - // Alternative: parse the json loaded to build the page https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions - // filter data.Catalog.searchStore.elements for .promotions.promotionalOffers being set and build URL with .catalogNs.mappings[0].pageSlug or .urlSlug if not set to some wrong id like it was the case for spirit-of-the-north-f58a66 - this is also what's done here: https://github.com/claabs/epicgames-freegames-node/blob/938a9653ffd08b8284ea32cf01ac8727d25c5d4c/src/puppet/free-games.ts#L138-L213 - const urlSlugs = await Promise.all((await game_loc.elementHandles()).map(a => a.getAttribute('href'))); - const urls = urlSlugs.map(s => 'https://store.epicgames.com' + s); - console.log('Free games:', urls); - - for (const url of urls) { - await page.goto(url); // , { waitUntil: 'domcontentloaded' }); - const btnText = await page.locator('//button[@data-testid="purchase-cta-button"][not(contains(.,"Loading"))]').first().innerText(); // barrier to block until page is loaded - - // click Continue if 'This game contains mature content recommended only for ages 18+' - if (await page.locator('button:has-text("Continue")').count() > 0) { - console.log(' This game contains mature content recommended only for ages 18+'); - await page.click('button:has-text("Continue")', { delay: 111 }); - await page.waitForTimeout(2000); - } - - const title = await page.locator('h1').first().innerText(); - const game_id = page.url().split('/').pop(); - db.data[user][game_id] ||= { title, time: datetime(), url: page.url() }; // this will be set on the initial run only! - console.log('Current free game:', title); + page.locator('button:has-text("Accept All Cookies")').click().catch(_ => { }); + for (const p of await page.locator('article.asset').all()) { + const link = p.locator('h3 a'); + const title = await link.innerText(); + const url = 'https://www.unrealengine.com' + await link.getAttribute('href'); + console.log(title, url); + const id = page.url().split('/').pop(); + db.data[user][id] ||= { title, time: datetime(), url }; // this will be set on the initial run only! const notify_game = { title, url, status: 'failed' }; notify_games.push(notify_game); // status is updated below - - if (btnText.toLowerCase() == 'in library') { - console.log(' Already in library! Nothing to claim.'); - notify_game.status = 'existed'; - db.data[user][game_id].status ||= 'existed'; // does not overwrite claimed or failed - if (db.data[user][game_id].status.startsWith('failed')) db.data[user][game_id].status = 'manual'; // was failed but now it's claimed - } else if (btnText.toLowerCase() == 'requires base game') { - console.log(' Requires base game! Nothing to claim.'); - notify_game.status = 'requires base game'; - db.data[user][game_id].status ||= 'failed:requires-base-game'; - // TODO claim base game if it is free - const baseUrl = 'https://store.epicgames.com' + await page.locator('a:has-text("Overview")').getAttribute('href'); - console.log(' Base game:', baseUrl); - // await page.click('a:has-text("Overview")'); - } else { // GET - console.log(' Not in library yet! Click GET.'); - await page.click('[data-testid="purchase-cta-button"]', { delay: 11 }); // got stuck here without delay (or mouse move), see #75, 1ms was also enough - - // click Continue if 'Device not supported. This product is not compatible with your current device.' - avoided by Windows userAgent? - page.click('button:has-text("Continue")').catch(_ => { }); // needed since change from Chromium to Firefox? - - // Accept End User License Agreement (only needed once) - page.locator('input#agree').waitFor().then(async () => { - console.log('Accept End User License Agreement (only needed once)'); - await page.locator('input#agree').check(); - await page.locator('button:has-text("Accept")').click(); - }).catch(_ => { }); - - // it then creates an iframe for the purchase - await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? - const iframe = page.frameLocator('#webPurchaseContainer iframe'); - // skip game if unavailable in region, https://github.com/vogler/free-games-claimer/issues/46 TODO check games for account's region - if (await iframe.locator(':has-text("unavailable in your region")').count() > 0) { - console.error(' This product is unavailable in your region!'); - db.data[user][game_id].status = notify_game.status = 'unavailable-in-region'; - continue; - } - - iframe.locator('.payment-pin-code').waitFor().then(async () => { - if (!cfg.eg_parentalpin) { - console.error(' EG_PARENTALPIN not set. Need to enter Parental Control PIN manually.'); - notify('epic-games: EG_PARENTALPIN not set. Need to enter Parental Control PIN manually.'); - } - await iframe.locator('input.payment-pin-code__input').first().type(cfg.eg_parentalpin); - await iframe.locator('button:has-text("Continue")').click({ delay: 11 }); - }).catch(_ => { }); - - if (cfg.debug) await page.pause(); - if (cfg.dryrun) { - console.log(' DRYRUN=1 -> Skip order!'); - continue; - } - - // Playwright clicked before button was ready to handle event, https://github.com/vogler/free-games-claimer/issues/84#issuecomment-1474346591 - await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 }); - - // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 - const btnAgree = iframe.locator('button:has-text("I Agree")'); - btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree' - try { - // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? - const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); - captcha.waitFor().then(async () => { // don't await, since element may not be shown - // console.info(' Got hcaptcha challenge! NopeCHA extension will likely solve it.') - console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') - // await page.waitForTimeout(2000); - // const p = path.resolve(cfg.dir.screenshots, 'epic-games', 'captcha', `${filenamify(datetime())}.png`); - // await captcha.screenshot({ path: p }); - // console.info(' Saved a screenshot of hcaptcha challenge to', p); - // console.error(' Got hcaptcha challenge. To avoid it, get a link from https://www.hcaptcha.com/accessibility'); // TODO save this link in config and visit it daily to set accessibility cookie to avoid captcha challenge? - }).catch(_ => { }); // may time out if not shown - await page.waitForSelector('text=Thanks for your order!'); - db.data[user][game_id].status = 'claimed'; - db.data[user][game_id].time = datetime(); // claimed time overwrites failed/dryrun time - console.log(' Claimed successfully!'); - // context.setDefaultTimeout(cfg.timeout); - } catch (e) { - console.log(e); - // console.error(' Failed to claim! Try again if NopeCHA timed out. Click the extension to see if you ran out of credits (refill after 24h). To avoid captchas try to get a new IP or set a cookie from https://www.hcaptcha.com/accessibility'); - console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); - const p = path.resolve(cfg.dir.screenshots, 'epic-games', 'failed', `${game_id}_${filenamify(datetime())}.png`); - await page.screenshot({ path: p, fullPage: true }); - db.data[user][game_id].status = 'failed'; - } - notify_game.status = db.data[user][game_id].status; // claimed or failed - - const p = path.resolve(cfg.dir.screenshots, 'epic-games', `${game_id}.png`); - if (!existsSync(p)) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... + if (await p.locator('.btn .in-cart').count()){ + console.log(' already in cart'); + continue; } + await p.locator('.btn .add').click(); + console.log(' added to cart'); } + const price = (await page.locator('.shopping-cart .total .price').innerText()).split(' '); + console.log('price: ', price[1], 'instead of', price[0]); + if (price[1] != '0') { + console.error('Price is not 0! Exit!'); + process.exit(1); + } + // await page.pause(); + console.log('Click shopping cart'); + await page.locator('.shopping-cart').click(); + // await page.waitForTimeout(2000); + await page.locator('button.checkout').click(); + console.log('Click checkout'); + // maybe: Accept End User License Agreement + page.locator('[name=accept-label]').check().then(() => { + console.log('Accept End User License Agreement'); + page.locator('span:text-is("Accept")').click() // otherwise matches 'Accept All Cookies' + }).catch(_ => { }); + // await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? + const iframe = page.frameLocator('#webPurchaseContainer iframe'); + + if (cfg.debug) await page.pause(); + if (cfg.dryrun) { + console.log(' DRYRUN=1 -> Skip order!'); + process.exit(); + } + + await iframe.locator('button:has-text("Place Order")').click(); + // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 + const btnAgree = iframe.locator('button:has-text("I Agree")'); + try { + context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? + await Promise.any([btnAgree.click(), page.waitForSelector('text=Thank you').then(_ => { })]); // EU: wait for agree button, non-EU: potentially done + + const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); + captcha.waitFor().then(async () => { // don't await, since element may not be shown + console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') + }).catch(_ => { }); // may time out if not shown + await page.waitForSelector('text=Thank you'); // EU: wait, non-EU: wait again = no-op + // db.data[user][id].status = 'claimed'; + // db.data[user][id].time = datetime(); // claimed time overwrites failed/dryrun time + notify_games.forEach(g => g.status = 'claimed'); + console.log(' Claimed successfully!'); + context.setDefaultTimeout(cfg.timeout); + } catch (e) { + console.log(e); + console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); + // const p = path.resolve(cfg.dir.screenshots, 'unrealengine', 'failed', `${id}_${filenamify(datetime())}.png`); + // await page.screenshot({ path: p, fullPage: true }); + // db.data[user][id].status = 'failed'; + notify_games.forEach(g => g.status = 'failed'); + } + + const p = path.resolve(cfg.dir.screenshots, 'unrealengine', `${filenamify(datetime())}.png`); + if (notify_games.length) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... + console.log('Done'); } catch (error) { console.error(error); // .toString()? process.exitCode ||= 1; if (error.message && process.exitCode != 130) - notify(`epic-games failed: ${error.message.split('\n')[0]}`); + notify(`unrealengine failed: ${error.message.split('\n')[0]}`); } finally { await db.write(); // write out json db - if (notify_games.filter(g => g.status != 'existed' && g.status != 'requires base game').length) { // don't notify if all were already claimed - notify(`epic-games (${user}):
${html_game_list(notify_games)}`); + if (notify_games.filter(g => g.status != 'existed').length) { // don't notify if all were already claimed + notify(`unrealengine (${user}):
${html_game_list(notify_games)}`); } } if (cfg.debug) writeFileSync(path.resolve(cfg.dir.browser, 'cookies.json'), JSON.stringify(await context.cookies())); -await context.close(); \ No newline at end of file +await context.close(); From 631197371f5bdfcfa7e1262e99bc667a1e53cf14 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Sat, 29 Apr 2023 11:01:16 +0200 Subject: [PATCH 4/8] ue: claimed successfully; set status, detect owned, better log --- unrealengine.js | 143 ++++++++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 60 deletions(-) diff --git a/unrealengine.js b/unrealengine.js index b7b7c63..25be32e 100644 --- a/unrealengine.js +++ b/unrealengine.js @@ -86,77 +86,100 @@ try { db.data[user] ||= {}; page.locator('button:has-text("Accept All Cookies")').click().catch(_ => { }); + + const ids = []; for (const p of await page.locator('article.asset').all()) { const link = p.locator('h3 a'); const title = await link.innerText(); const url = 'https://www.unrealengine.com' + await link.getAttribute('href'); - console.log(title, url); - const id = page.url().split('/').pop(); - db.data[user][id] ||= { title, time: datetime(), url }; // this will be set on the initial run only! + console.log([title, url]); + const id = url.split('/').pop(); + db.data[user][id] ||= { title, time: datetime(), url, status: 'failed' }; // this will be set on the initial run only! const notify_game = { title, url, status: 'failed' }; notify_games.push(notify_game); // status is updated below - if (await p.locator('.btn .in-cart').count()){ - console.log(' already in cart'); + // if (await p.locator('.btn .add-review-btn').count()) { // did not work + if((await p.getAttribute('class')).includes('asset--owned')) { + console.log(' ↳ Already claimed'); + if (db.data[user][id].status != 'claimed') { + db.data[user][id].status = 'existed'; + notify_game.status = 'existed'; + } + continue; + } + if (await p.locator('.btn .in-cart').count()) { + console.log(' ↳ Already in cart'); continue; } await p.locator('.btn .add').click(); - console.log(' added to cart'); + console.log(' ↳ Added to cart'); + ids.push(id); } - const price = (await page.locator('.shopping-cart .total .price').innerText()).split(' '); - console.log('price: ', price[1], 'instead of', price[0]); - if (price[1] != '0') { - console.error('Price is not 0! Exit!'); - process.exit(1); + if (!ids.length) { + console.log('Nothing to claim'); + } else { + const price = (await page.locator('.shopping-cart .total .price').innerText()).split(' '); + console.log('Price: ', price[1], 'instead of', price[0]); + if (price[1] != '0') { + console.error('Price is not 0! Exit!'); + process.exit(1); + } + // await page.pause(); + console.log('Click shopping cart'); + await page.locator('.shopping-cart').click(); + // await page.waitForTimeout(2000); + await page.locator('button.checkout').click(); + console.log('Click checkout'); + // maybe: Accept End User License Agreement + page.locator('[name=accept-label]').check().then(() => { + console.log('Accept End User License Agreement'); + page.locator('span:text-is("Accept")').click() // otherwise matches 'Accept All Cookies' + }).catch(_ => { }); + await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? + const iframe = page.frameLocator('#webPurchaseContainer iframe'); + + if (cfg.debug) await page.pause(); + if (cfg.dryrun) { + console.log('DRYRUN=1 -> Skip order!'); + throw new Error('DRYRUN=1'); + } + + console.log('Click Place Order'); + // Playwright clicked before button was ready to handle event, https://github.com/vogler/free-games-claimer/issues/84#issuecomment-1474346591 + await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 }); + + // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 + const btnAgree = iframe.locator('button:has-text("I Agree")'); + btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree' + try { + // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? + const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); + captcha.waitFor().then(async () => { // don't await, since element may not be shown + // console.info(' Got hcaptcha challenge! NopeCHA extension will likely solve it.') + console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') + }).catch(_ => { }); // may time out if not shown + await page.waitForSelector('text=Thank you'); + for (const id of ids) { + db.data[user][id].status = 'claimed'; + db.data[user][id].time = datetime(); // claimed time overwrites failed/dryrun time + } + notify_games.forEach(g => g.status == 'failed' && (g.status = 'claimed')); + console.log('Claimed successfully!'); + // context.setDefaultTimeout(cfg.timeout); + } catch (e) { + console.log(e); + // console.error(' Failed to claim! Try again if NopeCHA timed out. Click the extension to see if you ran out of credits (refill after 24h). To avoid captchas try to get a new IP or set a cookie from https://www.hcaptcha.com/accessibility'); + console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); + const p = path.resolve(cfg.dir.screenshots, 'unrealengine', 'failed', `${filenamify(datetime())}.png`); + await page.screenshot({ path: p, fullPage: true }); + // db.data[user][id].status = 'failed'; + notify_games.forEach(g => g.status = 'failed'); + } + // notify_game.status = db.data[user][game_id].status; // claimed or failed + + const p = path.resolve(cfg.dir.screenshots, 'unrealengine', `${filenamify(datetime())}.png`); + if (notify_games.length) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... + console.log('Done'); } - // await page.pause(); - console.log('Click shopping cart'); - await page.locator('.shopping-cart').click(); - // await page.waitForTimeout(2000); - await page.locator('button.checkout').click(); - console.log('Click checkout'); - // maybe: Accept End User License Agreement - page.locator('[name=accept-label]').check().then(() => { - console.log('Accept End User License Agreement'); - page.locator('span:text-is("Accept")').click() // otherwise matches 'Accept All Cookies' - }).catch(_ => { }); - // await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed? - const iframe = page.frameLocator('#webPurchaseContainer iframe'); - - if (cfg.debug) await page.pause(); - if (cfg.dryrun) { - console.log(' DRYRUN=1 -> Skip order!'); - process.exit(); - } - - await iframe.locator('button:has-text("Place Order")').click(); - // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872 - const btnAgree = iframe.locator('button:has-text("I Agree")'); - try { - context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s? - await Promise.any([btnAgree.click(), page.waitForSelector('text=Thank you').then(_ => { })]); // EU: wait for agree button, non-EU: potentially done - - const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe'); - captcha.waitFor().then(async () => { // don't await, since element may not be shown - console.error(' Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.') - }).catch(_ => { }); // may time out if not shown - await page.waitForSelector('text=Thank you'); // EU: wait, non-EU: wait again = no-op - // db.data[user][id].status = 'claimed'; - // db.data[user][id].time = datetime(); // claimed time overwrites failed/dryrun time - notify_games.forEach(g => g.status = 'claimed'); - console.log(' Claimed successfully!'); - context.setDefaultTimeout(cfg.timeout); - } catch (e) { - console.log(e); - console.error(' Failed to claim! To avoid captchas try to get a new IP address.'); - // const p = path.resolve(cfg.dir.screenshots, 'unrealengine', 'failed', `${id}_${filenamify(datetime())}.png`); - // await page.screenshot({ path: p, fullPage: true }); - // db.data[user][id].status = 'failed'; - notify_games.forEach(g => g.status = 'failed'); - } - - const p = path.resolve(cfg.dir.screenshots, 'unrealengine', `${filenamify(datetime())}.png`); - if (notify_games.length) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long... - console.log('Done'); } catch (error) { console.error(error); // .toString()? process.exitCode ||= 1; From ce51c269f8a1cfc24a18051a4a92cbc417f2fe70 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 5 May 2023 09:30:37 +0200 Subject: [PATCH 5/8] eg: only notify for status 'claimed' or 'failed'; DRYRUN -> 'skipped' --- epic-games.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/epic-games.js b/epic-games.js index 3819083..c38bdfc 100644 --- a/epic-games.js +++ b/epic-games.js @@ -177,6 +177,7 @@ try { if (cfg.debug) await page.pause(); if (cfg.dryrun) { console.log(' DRYRUN=1 -> Skip order!'); + notify_game.status = 'skipped'; continue; } @@ -224,9 +225,9 @@ try { notify(`epic-games failed: ${error.message.split('\n')[0]}`); } finally { await db.write(); // write out json db - if (notify_games.filter(g => g.status != 'existed' && g.status != 'requires base game').length) { // don't notify if all were already claimed + if (notify_games.filter(g => g.status == 'claimed' || g.status == 'failed').length) { // don't notify if all have status 'existed', 'manual', 'requires base game', 'unavailable-in-region', 'skipped' notify(`epic-games (${user}):
${html_game_list(notify_games)}`); } } if (cfg.debug) writeFileSync(path.resolve(cfg.dir.browser, 'cookies.json'), JSON.stringify(await context.cookies())); -await context.close(); \ No newline at end of file +await context.close(); From 066a99c77c9cc78baa71c2abe467385830269280 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 5 May 2023 09:54:21 +0200 Subject: [PATCH 6/8] eg: RECORD=1 to recordVideo & recordHar --- config.js | 1 + epic-games.js | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config.js b/config.js index 33016ca..32974e1 100644 --- a/config.js +++ b/config.js @@ -6,6 +6,7 @@ dotenv.config({ path: 'data/config.env' }); // loads env vars from file - will n // Options - also see table in README.md export const cfg = { debug: process.env.PWDEBUG == '1', // runs non-headless and opens https://playwright.dev/docs/inspector + record: process.env.RECORD == '1', // `recordHar` (network) + `recordVideo` dryrun: process.env.DRYRUN == '1', // don't claim anything show: process.env.SHOW == '1', // run non-headless get headless() { return !this.debug && !this.show }, diff --git a/epic-games.js b/epic-games.js index c38bdfc..a107a1f 100644 --- a/epic-games.js +++ b/epic-games.js @@ -27,7 +27,8 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, { userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 locale: "en-US", // ignore OS locale to be sure to have english text for locators - // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated + recordVideo: cfg.record && { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } }, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800 + recordHar: cfg.record && { path: `data/record/eg-${datetime()}.har` }, // will record a HAR file with network requests and responses; can be imported in Chrome devtools args: [ // https://peter.sh/experiments/chromium-command-line-switches // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' // '--restore-last-session', // does not apply for crash/killed @@ -45,6 +46,12 @@ if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist // console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); +if (cfg.record && cfg.debug) { + // const filter = _ => true; + const filter = r => r.url().includes('store.epicgames.com'); + page.on('request', request => filter(request) && console.log('>>', request.method(), request.url())); + page.on('response', response => filter(response) && console.log('<<', response.status(), response.url())); +} const notify_games = []; let user; From 7fc0fbc69c969d8de2fae7cc8cf6dd21711804d8 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Fri, 5 May 2023 13:36:51 +0200 Subject: [PATCH 7/8] Revert "eg: RECORD=1 to recordVideo & recordHar" This reverts commit 066a99c77c9cc78baa71c2abe467385830269280. --- config.js | 1 - epic-games.js | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/config.js b/config.js index 32974e1..33016ca 100644 --- a/config.js +++ b/config.js @@ -6,7 +6,6 @@ dotenv.config({ path: 'data/config.env' }); // loads env vars from file - will n // Options - also see table in README.md export const cfg = { debug: process.env.PWDEBUG == '1', // runs non-headless and opens https://playwright.dev/docs/inspector - record: process.env.RECORD == '1', // `recordHar` (network) + `recordVideo` dryrun: process.env.DRYRUN == '1', // don't claim anything show: process.env.SHOW == '1', // run non-headless get headless() { return !this.debug && !this.show }, diff --git a/epic-games.js b/epic-games.js index a107a1f..c38bdfc 100644 --- a/epic-games.js +++ b/epic-games.js @@ -27,8 +27,7 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, { userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 locale: "en-US", // ignore OS locale to be sure to have english text for locators - recordVideo: cfg.record && { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } }, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800 - recordHar: cfg.record && { path: `data/record/eg-${datetime()}.har` }, // will record a HAR file with network requests and responses; can be imported in Chrome devtools + // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated args: [ // https://peter.sh/experiments/chromium-command-line-switches // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' // '--restore-last-session', // does not apply for crash/killed @@ -46,12 +45,6 @@ if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist // console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); -if (cfg.record && cfg.debug) { - // const filter = _ => true; - const filter = r => r.url().includes('store.epicgames.com'); - page.on('request', request => filter(request) && console.log('>>', request.method(), request.url())); - page.on('response', response => filter(response) && console.log('<<', response.status(), response.url())); -} const notify_games = []; let user; From 8f174c4bf095f604829407d5415dde7324f334f1 Mon Sep 17 00:00:00 2001 From: Ralf Vogler Date: Mon, 8 May 2023 17:26:38 +0200 Subject: [PATCH 8/8] eg: RECORD=1 to recordVideo & recordHar; fixed: `recordVideo` can't be false Strangely `recordHar` can be false instead of undefined, but made it symmetric. --- config.js | 1 + epic-games.js | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config.js b/config.js index 33016ca..32974e1 100644 --- a/config.js +++ b/config.js @@ -6,6 +6,7 @@ dotenv.config({ path: 'data/config.env' }); // loads env vars from file - will n // Options - also see table in README.md export const cfg = { debug: process.env.PWDEBUG == '1', // runs non-headless and opens https://playwright.dev/docs/inspector + record: process.env.RECORD == '1', // `recordHar` (network) + `recordVideo` dryrun: process.env.DRYRUN == '1', // don't claim anything show: process.env.SHOW == '1', // run non-headless get headless() { return !this.debug && !this.show }, diff --git a/epic-games.js b/epic-games.js index c38bdfc..7e1bf62 100644 --- a/epic-games.js +++ b/epic-games.js @@ -27,7 +27,8 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, { userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated? // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0 locale: "en-US", // ignore OS locale to be sure to have english text for locators - // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated + recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800 + recordHar: cfg.record ? { path: `data/record/eg-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools args: [ // https://peter.sh/experiments/chromium-command-line-switches // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.' // '--restore-last-session', // does not apply for crash/killed @@ -45,6 +46,12 @@ if (!cfg.debug) context.setDefaultTimeout(cfg.timeout); const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist // console.debug('userAgent:', await page.evaluate(() => navigator.userAgent)); +if (cfg.record && cfg.debug) { + // const filter = _ => true; + const filter = r => r.url().includes('store.epicgames.com'); + page.on('request', request => filter(request) && console.log('>>', request.method(), request.url())); + page.on('response', response => filter(response) && console.log('<<', response.status(), response.url())); +} const notify_games = []; let user;