Merge branch 'main' into urlsFromJson

config.js

@@ -6,6 +6,7 @@ dotenv.config({ path: 'data/config.env' }); // loads env vars from file - will n
 // Options - also see table in README.md
 export const cfg = {
   debug: process.env.PWDEBUG == '1', // runs non-headless and opens https://playwright.dev/docs/inspector
+  record: process.env.RECORD == '1', // `recordHar` (network) + `recordVideo`
   dryrun: process.env.DRYRUN == '1', // don't claim anything
   show: process.env.SHOW == '1', // run non-headless
   get headless() { return !this.debug && !this.show },

epic-games.js

@@ -40,7 +40,8 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, {
   userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated?
   // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0
   locale: "en-US", // ignore OS locale to be sure to have english text for locators
-  // recordVideo: { dir: 'data/videos/' }, // will record a .webm video for each page navigated
+  recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
+  recordHar: cfg.record ? { path: `data/record/eg-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
   args: [ // https://peter.sh/experiments/chromium-command-line-switches
     // don't want to see bubble 'Restore pages? Chrome didn't shut down correctly.'
     // '--restore-last-session', // does not apply for crash/killed
@@ -58,6 +59,12 @@ if (!cfg.debug) context.setDefaultTimeout(cfg.timeout);
 
 const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist
 // console.debug('userAgent:', await page.evaluate(() => navigator.userAgent));
+if (cfg.record && cfg.debug) {
+  // const filter = _ => true;
+  const filter = r => r.url().includes('store.epicgames.com');
+  page.on('request', request => filter(request) && console.log('>>', request.method(), request.url()));
+  page.on('response', response => filter(response) && console.log('<<', response.status(), response.url()));
+}
 
 const notify_games = [];
 let user;
@@ -182,6 +189,7 @@ try {
       if (cfg.debug) await page.pause();
       if (cfg.dryrun) {
         console.log('  DRYRUN=1 -> Skip order!');
+        notify_game.status = 'skipped';
         continue;
       }
 
@@ -232,7 +240,7 @@ try {
     notify(`epic-games failed: ${error.message.split('\n')[0]}`);
 } finally {
   await db.write(); // write out json db
-  if (notify_games.filter(g => g.status != 'existed' && g.status != 'requires base game').length) { // don't notify if all were already claimed
+  if (notify_games.filter(g => g.status == 'claimed' || g.status == 'failed').length) { // don't notify if all have status 'existed', 'manual', 'requires base game', 'unavailable-in-region', 'skipped'
     notify(`epic-games (${user}):<br>${html_game_list(notify_games)}`);
   }
 }

migrate.js

@@ -0,0 +1,34 @@
+import { existsSync } from 'fs';
+import { Low } from 'lowdb';
+import { JSONFile } from 'lowdb/node';
+import { datetime } from './util.js';
+
+const datetime_UTCtoLocalTimezone = async file => {
+  if (!existsSync(file))
+    return console.error('File does not exist:', file);
+  const db = new Low(new JSONFile(file));
+  await db.read();
+  db.data ||= {};
+  console.log('Migrating', file);
+  for (const user in db.data) {
+    for (const game in db.data[user]) {
+      const time1 = db.data[user][game].time;
+      const time1s = time1.endsWith('Z') ? time1 : time1 + ' UTC';
+      const time2 = datetime(new Date(time1s));
+      console.log([game, time1, time2]);
+      db.data[user][game].time = time2;
+    }
+  }
+  // console.log(db.data);
+  await db.write(); // write out json db
+};
+
+const args = process.argv.slice(2);
+if (args[0] == 'localtime') {
+  const files = args.slice(1);
+  console.log('Will convert UTC datetime to local timezone for', files);
+  files.forEach(datetime_UTCtoLocalTimezone);
+} else {
+  console.log('Usage: node migrate.js <cmd> <args>');
+  console.log('       node migrate.js localtime data/*.json');
+}

unrealengine.js

@@ -0,0 +1,195 @@
+// TODO This is mostly a copy of epic-games.js
+
+import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra
+import { authenticator } from 'otplib';
+import path from 'path';
+import { existsSync, writeFileSync } from 'fs';
+import { jsonDb, datetime, stealth, filenamify, prompt, notify, html_game_list, handleSIGINT } from './util.js';
+import { cfg } from './config.js';
+
+const URL_CLAIM = 'https://www.unrealengine.com/marketplace/en-US/assets?count=20&sortBy=effectiveDate&sortDir=DESC&start=0&tag=4910';
+const URL_LOGIN = 'https://www.epicgames.com/id/login?lang=en-US&noHostRedirect=true&redirectUrl=' + URL_CLAIM;
+
+console.log(datetime(), 'started checking unrealengine');
+
+const db = await jsonDb('unrealengine.json');
+db.data ||= {};
+
+handleSIGINT();
+
+// https://playwright.dev/docs/auth#multi-factor-authentication
+const context = await firefox.launchPersistentContext(cfg.dir.browser, {
+  headless: cfg.headless,
+  viewport: { width: cfg.width, height: cfg.height },
+  userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated?
+  // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0
+  locale: "en-US", // ignore OS locale to be sure to have english text for locators
+});
+
+await stealth(context);
+
+if (!cfg.debug) context.setDefaultTimeout(cfg.timeout);
+
+const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist
+// console.debug('userAgent:', await page.evaluate(() => navigator.userAgent));
+
+const notify_games = [];
+let user;
+
+try {
+  await context.addCookies([{name: 'OptanonAlertBoxClosed', value: new Date(Date.now() - 5*24*60*60*1000).toISOString(), domain: '.epicgames.com', path: '/'}]); // Accept cookies to get rid of banner to save space on screen. Set accept time to 5 days ago.
+
+  await page.goto(URL_CLAIM, { waitUntil: 'domcontentloaded' }); // 'domcontentloaded' faster than default 'load' https://playwright.dev/docs/api/class-page#page-goto
+
+  while (await page.locator('a[role="button"]:has-text("Sign In")').count() > 0) {
+    console.error('Not signed in anymore. Please login in the browser or here in the terminal.');
+    if (cfg.novnc_port) console.info(`Open http://localhost:${cfg.novnc_port} to login inside the docker container.`);
+    if (!cfg.debug) context.setDefaultTimeout(cfg.login_timeout); // give user some extra time to log in
+    console.info(`Login timeout is ${cfg.login_timeout/1000} seconds!`);
+    await page.goto(URL_LOGIN, { waitUntil: 'domcontentloaded' });
+    if (cfg.eg_email && cfg.eg_password) console.info('Using email and password from environment.');
+    else console.info('Press ESC to skip the prompts if you want to login in the browser (not possible in headless mode).');
+    const email = cfg.eg_email || await prompt({message: 'Enter email'});
+    const password = email && (cfg.eg_password || await prompt({type: 'password', message: 'Enter password'}));
+    if (email && password) {
+      await page.click('text=Sign in with Epic Games');
+      await page.fill('#email', email);
+      await page.fill('#password', password);
+      await page.click('button[type="submit"]');
+      page.waitForSelector('#h_captcha_challenge_login_prod iframe').then(() => {
+        console.error('Got a captcha during login (likely due to too many attempts)! You may solve it in the browser, get a new IP or try again in a few hours.');
+        notify('unrealengine: got captcha during login. Please check.');
+      }).catch(_ => { });
+      // handle MFA, but don't await it
+      page.waitForURL('**/id/login/mfa**').then(async () => {
+        console.log('Enter the security code to continue - This appears to be a new device, browser or location. A security code has been sent to your email address at ...');
+        // TODO locator for text (email or app?)
+        const otp = cfg.eg_otpkey && authenticator.generate(cfg.eg_otpkey) || await prompt({type: 'text', message: 'Enter two-factor sign in code', validate: n => n.toString().length == 6 || 'The code must be 6 digits!'}); // can't use type: 'number' since it strips away leading zeros and codes sometimes have them
+        await page.type('input[name="code-input-0"]', otp.toString());
+        await page.click('button[type="submit"]');
+      }).catch(_ => { });
+    } else {
+      console.log('Waiting for you to login in the browser.');
+      await notify('unrealengine: no longer signed in and not enough options set for automatic login.');
+      if (cfg.headless) {
+        console.log('Run `SHOW=1 node unrealengine` to login in the opened browser.');
+        await context.close(); // finishes potential recording
+        process.exit(1);
+      }
+    }
+    await page.waitForURL(URL_CLAIM);
+    if (!cfg.debug) context.setDefaultTimeout(cfg.timeout);
+  }
+  await page.waitForTimeout(1000);
+  user = await page.locator('.user-label').first().innerHTML();
+  console.log(`Signed in as ${user}`);
+  db.data[user] ||= {};
+
+  page.locator('button:has-text("Accept All Cookies")').click().catch(_ => { });
+
+  const ids = [];
+  for (const p of await page.locator('article.asset').all()) {
+    const link = p.locator('h3 a');
+    const title = await link.innerText();
+    const url = 'https://www.unrealengine.com' + await link.getAttribute('href');
+    console.log([title, url]);
+    const id = url.split('/').pop();
+    db.data[user][id] ||= { title, time: datetime(), url, status: 'failed' }; // this will be set on the initial run only!
+    const notify_game = { title, url, status: 'failed' };
+    notify_games.push(notify_game); // status is updated below
+    // if (await p.locator('.btn .add-review-btn').count()) { // did not work
+    if((await p.getAttribute('class')).includes('asset--owned')) {
+      console.log('  ↳ Already claimed');
+      if (db.data[user][id].status != 'claimed') {
+        db.data[user][id].status = 'existed';
+        notify_game.status = 'existed';
+      }
+      continue;
+    }
+    if (await p.locator('.btn .in-cart').count()) {
+      console.log('  ↳ Already in cart');
+      continue;
+    }
+    await p.locator('.btn .add').click();
+    console.log('  ↳ Added to cart');
+    ids.push(id);
+  }
+  if (!ids.length) {
+    console.log('Nothing to claim');
+  } else {
+    const price = (await page.locator('.shopping-cart .total .price').innerText()).split(' ');
+    console.log('Price: ', price[1], 'instead of', price[0]);
+    if (price[1] != '0') {
+      console.error('Price is not 0! Exit!');
+      process.exit(1);
+    }
+    // await page.pause();
+    console.log('Click shopping cart');
+    await page.locator('.shopping-cart').click();
+    // await page.waitForTimeout(2000);
+    await page.locator('button.checkout').click();
+    console.log('Click checkout');
+    // maybe: Accept End User License Agreement
+    page.locator('[name=accept-label]').check().then(() => {
+      console.log('Accept End User License Agreement');
+      page.locator('span:text-is("Accept")').click() // otherwise matches 'Accept All Cookies'
+    }).catch(_ => { });
+    await page.waitForSelector('#webPurchaseContainer iframe'); // TODO needed?
+    const iframe = page.frameLocator('#webPurchaseContainer iframe');
+
+    if (cfg.debug) await page.pause();
+    if (cfg.dryrun) {
+      console.log('DRYRUN=1 -> Skip order!');
+      throw new Error('DRYRUN=1');
+    }
+
+    console.log('Click Place Order');
+    // Playwright clicked before button was ready to handle event, https://github.com/vogler/free-games-claimer/issues/84#issuecomment-1474346591
+    await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 });
+
+    // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872
+    const btnAgree = iframe.locator('button:has-text("I Agree")');
+    btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree'
+    try {
+      // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s?
+      const captcha = iframe.locator('#h_captcha_challenge_checkout_free_prod iframe');
+      captcha.waitFor().then(async () => { // don't await, since element may not be shown
+        // console.info('  Got hcaptcha challenge! NopeCHA extension will likely solve it.')
+        console.error('  Got hcaptcha challenge! Lost trust due to too many login attempts? You can solve the captcha in the browser or get a new IP address.')
+      }).catch(_ => { }); // may time out if not shown
+      await page.waitForSelector('text=Thank you');
+      for (const id of ids) {
+        db.data[user][id].status = 'claimed';
+        db.data[user][id].time = datetime(); // claimed time overwrites failed/dryrun time
+      }
+      notify_games.forEach(g => g.status == 'failed' && (g.status = 'claimed'));
+      console.log('Claimed successfully!');
+      // context.setDefaultTimeout(cfg.timeout);
+    } catch (e) {
+      console.log(e);
+      // console.error('  Failed to claim! Try again if NopeCHA timed out. Click the extension to see if you ran out of credits (refill after 24h). To avoid captchas try to get a new IP or set a cookie from https://www.hcaptcha.com/accessibility');
+      console.error('  Failed to claim! To avoid captchas try to get a new IP address.');
+      const p = path.resolve(cfg.dir.screenshots, 'unrealengine', 'failed', `${filenamify(datetime())}.png`);
+      await page.screenshot({ path: p, fullPage: true });
+      // db.data[user][id].status = 'failed';
+      notify_games.forEach(g => g.status = 'failed');
+    }
+    // notify_game.status = db.data[user][game_id].status; // claimed or failed
+
+    const p = path.resolve(cfg.dir.screenshots, 'unrealengine', `${filenamify(datetime())}.png`);
+    if (notify_games.length) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long...
+    console.log('Done');
+  }
+} catch (error) {
+  console.error(error); // .toString()?
+  process.exitCode ||= 1;
+  if (error.message && process.exitCode != 130)
+    notify(`unrealengine failed: ${error.message.split('\n')[0]}`);
+} finally {
+  await db.write(); // write out json db
+  if (notify_games.filter(g => g.status != 'existed').length) { // don't notify if all were already claimed
+    notify(`unrealengine (${user}):<br>${html_game_list(notify_games)}`);
+  }
+}
+if (cfg.debug) writeFileSync(path.resolve(cfg.dir.browser, 'cookies.json'), JSON.stringify(await context.cookies()));
+await context.close();

util.js

@@ -19,9 +19,9 @@ export const jsonDb = async file => {
 
 export const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 // date and time as UTC (no timezone offset) in nicely readable and sortable format, e.g., 2022-10-06 12:05:27.313
-export const datetime = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', '');
+export const datetimeUTC = (d = new Date()) => d.toISOString().replace('T', ' ').replace('Z', '');
-// same as datetime() but for local timezone, e.g., UTC + 2h for the above in DE
+// same as datetimeUTC() but for local timezone, e.g., UTC + 2h for the above in DE
-export const datetimeLocal = (d = new Date()) => datetime(new Date(d.getTime() - new Date().getTimezoneOffset() * 60000));
+export const datetime = (d = new Date()) => datetimeUTC(new Date(d.getTime() - d.getTimezoneOffset() * 60000));
 export const filenamify = s => s.replaceAll(':', '.').replace(/[^a-z0-9 _\-.]/gi, '_'); // alternative: https://www.npmjs.com/package/filenamify - On Unix-like systems, / is reserved. On Windows, <>:"/\|?* along with trailing periods are reserved.
 
 export const handleSIGINT = () => process.on('SIGINT', () => { // e.g. when killed by Ctrl-C