Merge branch 'main' of https://github.com/vogler/free-games-claimer

.github/workflows/docker.yml

@@ -27,12 +27,13 @@ jobs:
       -
         name: Set environment variables
         run: |
-          echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+          BRANCH="${GITHUB_REF#refs/heads/}"
+          echo "BRANCH=$BRANCH" >> $GITHUB_ENV
           echo "NOW=$(date -R)" >> $GITHUB_ENV # date -Iseconds; date +'%Y-%m-%dT%H:%M:%S'
-          if [[ "${{ env.BRANCH }}" == "main" ]]; then
+          if [[ "$BRANCH" == "main" ]]; then
             echo "IMAGE_TAG=latest" >> $GITHUB_ENV
           else
-            echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+            echo "IMAGE_TAG=$BRANCH" >> $GITHUB_ENV
           fi
       -
         name: Set up QEMU

README.md

@@ -92,6 +92,7 @@ Available options/variables and their default values:
 | GOG_EMAIL     	|         	| GOG email for login. Overrides EMAIL.                                  	|
 | GOG_PASSWORD  	|         	| GOG password for login. Overrides PASSWORD.                            	|
 | GOG_NEWSLETTER	| 0       	| Do not unsubscribe from newsletter after claiming a game if 1.         	|
+| LG_EMAIL        |         	| Legacy Games: email to use for redeeming (if not set, defaults to PG_EMAIL)  |
 
 See `src/config.js` for all options.
 
@@ -103,7 +104,7 @@ You can pass variables using `-e VAR=VAL`, for example `docker run -e EMAIL=foo@
 If you are using [docker compose](https://docs.docker.com/compose/environment-variables/) (or Portainer etc.), you can put options in the `environment:` section.
 
 ##### Without Docker
-On Linux/macOS you can prefix the variables you want to set, for example `EMAIL=foo@bar.baz SHOW=1 node epic-games` will show the browser and skip asking you for your login email.
+On Linux/macOS you can prefix the variables you want to set, for example `EMAIL=foo@bar.baz SHOW=1 node epic-games` will show the browser and skip asking you for your login email. On Windows you have to use `set`, [example](https://github.com/vogler/free-games-claimer/issues/314).
 You can also put options in `data/config.env` which will be loaded by [dotenv](https://github.com/motdotla/dotenv).
 
 ### Notifications
@@ -156,7 +157,7 @@ If you want it to run regularly, you have to schedule the runs yourself:
 
 - Linux/macOS: `crontab -e` ([example](https://github.com/vogler/free-games-claimer/discussions/56))
 - macOS: [launchd](https://stackoverflow.com/questions/132955/how-do-i-set-a-task-to-run-every-so-often)
-- Windows: [task scheduler](https://active-directory-wp.com/docs/Usage/How_to_add_a_cron_job_on_Windows/Scheduled_tasks_and_cron_jobs_on_Windows/index.html), [other options](https://stackoverflow.com/questions/132971/what-is-the-windows-version-of-cron), or just put the command in a `.bat` file in Autostart if you restart often...
+- Windows: [task scheduler](https://active-directory-wp.com/docs/Usage/How_to_add_a_cron_job_on_Windows/Scheduled_tasks_and_cron_jobs_on_Windows/index.html) ([example](https://github.com/vogler/free-games-claimer/wiki/%5BHowTo%5D-Schedule-runs-on-Windows)), [other options](https://stackoverflow.com/questions/132971/what-is-the-windows-version-of-cron), or just put the command in a `.bat` file in Autostart if you restart often...
 - any OS: use a process manager like [pm2](https://pm2.keymetrics.io/docs/usage/restart-strategies/)
 - Docker Compose `command: bash -c "node epic-games; node prime-gaming; node gog; echo sleeping; sleep 1d"` additionally add `restart: unless-stopped` to it.
 

aliexpress.js

@@ -0,0 +1,105 @@
+import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra
+import { datetime, filenamify, prompt, handleSIGINT, stealth } from './src/util.js';
+import { cfg } from './src/config.js';
+
+const context = await firefox.launchPersistentContext(cfg.dir.browser, {
+  headless: cfg.headless,
+  viewport: { width: cfg.width, height: cfg.height },
+  locale: 'en-US', // ignore OS locale to be sure to have english text for locators -> done via /en in URL
+  recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
+  recordHar: cfg.record ? { path: `data/record/gog-${filenamify(datetime())}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
+  handleSIGINT: false, // have to handle ourselves and call context.close(), otherwise recordings from above won't be saved
+});
+handleSIGINT(context);
+await stealth(context);
+
+context.setDefaultTimeout(cfg.debug ? 0 : cfg.timeout);
+
+const page = context.pages().length ? context.pages()[0] : await context.newPage(); // should always exist
+
+const auth = async (url) => {
+  console.log('auth', url);
+  await page.goto(url, { waitUntil: 'domcontentloaded' });
+  // redirects to https://login.aliexpress.com/?return_url=https%3A%2F%2Fwww.aliexpress.com%2Fp%2Fcoin-pc-index%2Findex.html
+  await Promise.any([page.waitForURL(/.*login.aliexpress.com.*/).then(async () => {
+    // manual login
+    console.error('Not logged in! Will wait for 120s for you to login...');
+    // await page.waitForTimeout(120*1000);
+    // or try automated
+    page.locator('span:has-text("Switch account")').click().catch(_ => {}); // sometimes no longer logged in, but previous user/email is pre-selected -> in this case we want to go back to the classic login
+    const login = page.locator('.login-container');
+    const email = cfg.ae_email || await prompt({ message: 'Enter email' });
+    const emailInput = login.locator('input[label="Email or phone number"]');
+    await emailInput.fill(email);
+    await emailInput.blur(); // otherwise Continue button stays disabled
+    const continueButton = login.locator('button:has-text("Continue")');
+    await continueButton.click({ force: true }); // normal click waits for button to no longer be covered by their suggestion menu, so we have to force click somewhere for the menu to close and then click
+    await continueButton.click();
+    const password = email && (cfg.ae_password || await prompt({ type: 'password', message: 'Enter password' }));
+    await login.locator('input[label="Password"]').fill(password);
+    await login.locator('button:has-text("Sign in")').click();
+    const error = login.locator('.error-text');
+    error.waitFor().then(async _ => console.error('Login error:', await error.innerText()));
+    await page.waitForURL(url);
+    // await page.addLocatorHandler(page.getByRole('button', { name: 'Accept cookies' }), btn => btn.click());
+    page.getByRole('button', { name: 'Accept cookies' }).click().then(_ => console.log('Accepted cookies')).catch(_ => { });
+  }), page.locator('#nav-user-account').waitFor()]).catch(_ => {});
+
+  // await page.locator('#nav-user-account').hover();
+  // console.log('Logged in as:', await page.locator('.welcome-name').innerText());
+};
+
+// copied URLs from AliExpress app on tablet which has menu for the used webview
+const urls = {
+  // works with desktop view, but stuck at 100% loading in mobile view:
+  coins: 'https://www.aliexpress.com/p/coin-pc-index/index.html',
+  // only work with mobile view:
+  grow: 'https://m.aliexpress.com/p/ae_fruit/index.html', // firefox: stuck at 60% loading, chrome: loads, but canvas
+  gogo: 'https://m.aliexpress.com/p/gogo-match-cc/index.html', // closes firefox?!
+  // only show notification to install the app
+  euro: 'https://m.aliexpress.com/p/european-cup/index.html', // doesn't load
+  merge: 'https://m.aliexpress.com/p/merge-market/index.html',
+};
+
+const coins = async () => {
+  // await auth(urls.coins);
+  await Promise.any([page.locator('.checkin-button').click(), page.locator('.addcoin').waitFor()]);
+  console.log('Coins:', await page.locator('.mycoin-content-right-money').innerText());
+  console.log('Streak:', await page.locator('.title-box').innerText());
+  console.log('Tomorrow:', await page.locator('.addcoin').innerText());
+};
+
+const grow = async () => {
+  await page.pause();
+};
+
+const gogo = async () => {
+  await page.pause();
+};
+
+const euro = async () => {
+  await page.pause();
+};
+
+const merge = async () => {
+  await page.pause();
+};
+
+try {
+  // await coins();
+  await [
+    coins,
+    // grow,
+    // gogo,
+    // euro,
+    // merge,
+  ].reduce((a, f) => a.then(async _ => { await auth(urls[f.name]); await f(); console.log() }), Promise.resolve());
+
+  // await page.pause();
+} catch (error) {
+  process.exitCode ||= 1;
+  console.error('--- Exception:');
+  console.error(error); // .toString()?
+}
+if (page.video()) console.log('Recorded video:', await page.video().path());
+await context.close();

epic-games.js

@@ -1,7 +1,7 @@
 import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra
 import { authenticator } from 'otplib';
 import path from 'path';
-import { existsSync, writeFileSync } from 'fs';
+import { existsSync, writeFileSync, appendFileSync } from 'fs';
 import { resolve, jsonDb, datetime, stealth, filenamify, prompt, notify, html_game_list, handleSIGINT } from './src/util.js';
 import { cfg } from './src/config.js';
 
@@ -16,16 +16,24 @@ const db = await jsonDb('epic-games.json', {});
 
 if (cfg.time) console.time('startup');
 
+const browserPrefs = path.join(cfg.dir.browser, 'prefs.js');
+if (existsSync(browserPrefs)) {
+  console.log('Adding webgl.disabled to', browserPrefs);
+  appendFileSync(browserPrefs, 'user_pref("webgl.disabled", true);'); // apparently Firefox removes duplicates (and sorts), so no problem appending every time
+} else {
+  console.log(browserPrefs, 'does not exist yet, will patch it on next run. Restart the script if you get a captcha.');
+}
+
 // https://playwright.dev/docs/auth#multi-factor-authentication
 const context = await firefox.launchPersistentContext(cfg.dir.browser, {
   headless: cfg.headless,
   viewport: { width: cfg.width, height: cfg.height },
-  userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated?
+  userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0', // see replace of Headless in util.newStealthContext. TODO Windows UA enough to avoid 'device not supported'? update if browser is updated?
   // userAgent firefox (macOS): Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0
   // userAgent firefox (docker): Mozilla/5.0 (X11; Linux aarch64; rv:109.0) Gecko/20100101 Firefox/115.0
   locale: 'en-US', // ignore OS locale to be sure to have english text for locators
   recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
-  recordHar: cfg.record ? { path: `data/record/eg-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
+  recordHar: cfg.record ? { path: `data/record/eg-${filenamify(datetime())}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
   handleSIGINT: false, // have to handle ourselves and call context.close(), otherwise recordings from above won't be saved
   // user settings for firefox have to be put in $BROWSER_DIR/user.js
   args: [ // https://wiki.mozilla.org/Firefox/CommandLineOptions
@@ -90,19 +98,26 @@ try {
     if (!email) await notifyBrowserLogin();
     else {
       // await page.click('text=Sign in with Epic Games');
-      await page.fill('#email', email);
-      await page.click('button[type="submit"]');
       page.waitForSelector('.h_captcha_challenge iframe').then(async () => {
         console.error('Got a captcha during login (likely due to too many attempts)! You may solve it in the browser, get a new IP or try again in a few hours.');
         await notify('epic-games: got captcha during login. Please check.');
       }).catch(_ => { });
       page.waitForSelector('p:has-text("Incorrect response.")').then(async () => {
-        console.error('Incorrect repsonse for captcha!');
+        console.error('Incorrect response for captcha!');
       }).catch(_ => { });
+      await page.fill('#email', email);
+      // await page.click('button[type="submit"]'); login was split in two steps for some time, now email and password are on the same form again
       const password = email && (cfg.eg_password || await prompt({ type: 'password', message: 'Enter password' }));
       if (!password) await notifyBrowserLogin();
+      else {
         await page.fill('#password', password);
         await page.click('button[type="submit"]');
+      }
+      const error = page.locator('#form-error-message');
+      error.waitFor().then(async () => {
+        console.error('Login error:', await error.innerText());
+        console.log('Please login in the browser!');
+      }).catch(_ => { });
       // handle MFA, but don't await it
       page.waitForURL('**/id/login/mfa**').then(async () => {
         console.log('Enter the security code to continue - This appears to be a new device, browser or location. A security code has been sent to your email address at ...');
@@ -141,7 +156,8 @@ try {
   for (const url of urls) {
     if (cfg.time) console.time('claim game');
     await page.goto(url); // , { waitUntil: 'domcontentloaded' });
-    const btnText = await page.locator('//button[@data-testid="purchase-cta-button"][not(contains(.,"Loading"))]').first().innerText(); // barrier to block until page is loaded
+    const purcahseBtn = page.locator('aside button').first();
+    const btnText = (await purcahseBtn.innerText()).toLowerCase(); // barrier to block until page is loaded
 
     // click Continue if 'This game contains mature content recommended only for ages 18+'
     if (await page.locator('button:has-text("Continue")').count() > 0) {
@@ -159,19 +175,25 @@ try {
       await page.waitForTimeout(2000);
     }
 
-    const title = await page.locator('h1').first().innerText();
+    let title;
+    if (await page.locator('span:text-is("About Bundle")').count()) {
+      // console.log('  This is a bundle containing: TODO');
+      title = (await page.locator('span:has-text("Buy"):left-of([data-testid="purchase-cta-button"])').first().innerText()).replace('Buy ', '');
+    } else {
+      title = await page.locator('h1').first().innerText();
+    }
     const game_id = page.url().split('/').pop();
     db.data[user][game_id] ||= { title, time: datetime(), url: page.url() }; // this will be set on the initial run only!
     console.log('Current free game:', title);
     const notify_game = { title, url, status: 'failed' };
     notify_games.push(notify_game); // status is updated below
 
-    if (btnText.toLowerCase() == 'in library') {
+    if (btnText == 'in library') {
       console.log('  Already in library! Nothing to claim.');
       notify_game.status = 'existed';
       db.data[user][game_id].status ||= 'existed'; // does not overwrite claimed or failed
       if (db.data[user][game_id].status.startsWith('failed')) db.data[user][game_id].status = 'manual'; // was failed but now it's claimed
-    } else if (btnText.toLowerCase() == 'requires base game') {
+    } else if (btnText == 'requires base game') {
       console.log('  Requires base game! Nothing to claim.');
       notify_game.status = 'requires base game';
       db.data[user][game_id].status ||= 'failed:requires-base-game';
@@ -179,9 +201,12 @@ try {
       const baseUrl = 'https://store.epicgames.com' + await page.locator('a:has-text("Overview")').getAttribute('href');
       console.log('  Base game:', baseUrl);
       // await page.click('a:has-text("Overview")');
+      // TODO handle this via function call for base game above since this will never terminate if DRYRUN=1
+      urls.push(baseUrl); // add base game to the list of games to claim
+      urls.push(url); // add add-on itself again
     } else { // GET
       console.log('  Not in library yet! Click GET.');
-      await page.click('[data-testid="purchase-cta-button"]', { delay: 11 }); // got stuck here without delay (or mouse move), see #75, 1ms was also enough
+      await purcahseBtn.click({ delay: 11 }); // got stuck here without delay (or mouse move), see #75, 1ms was also enough
 
       // click Continue if 'Device not supported. This product is not compatible with your current device.' - avoided by Windows userAgent?
       page.click('button:has-text("Continue")').catch(_ => { }); // needed since change from Chromium to Firefox?
@@ -192,7 +217,7 @@ try {
       // Accept End User License Agreement (only needed once)
       page.locator('input#agree').waitFor().then(async () => {
         console.log('  Accept End User License Agreement (only needed once)');
-        await page.locator('input#agree').check();
+        await page.locator('input#agree').check(); // TODO Bundle: got stuck here
         await page.locator('button:has-text("Accept")').click();
       }).catch(_ => { });
 
@@ -228,7 +253,7 @@ try {
       await iframe.locator('button:has-text("Place Order"):not(:has(.payment-loading--loading))').click({ delay: 11 });
 
       // I Agree button is only shown for EU accounts! https://github.com/vogler/free-games-claimer/pull/7#issuecomment-1038964872
-      const btnAgree = iframe.locator('button:has-text("I Agree")');
+      const btnAgree = iframe.locator('button:has-text("I Accept")');
       btnAgree.waitFor().then(() => btnAgree.click()).catch(_ => { }); // EU: wait for and click 'I Agree'
       try {
         // context.setDefaultTimeout(100 * 1000); // give time to solve captcha, iframe goes blank after 60s?
@@ -243,7 +268,11 @@ try {
           // console.info('  Saved a screenshot of hcaptcha challenge to', p);
           // console.error('  Got hcaptcha challenge. To avoid it, get a link from https://www.hcaptcha.com/accessibility'); // TODO save this link in config and visit it daily to set accessibility cookie to avoid captcha challenge?
         }).catch(_ => { }); // may time out if not shown
-        await page.locator('text=Thanks for your order!').waitFor({ state: 'attached' });
+        iframe.locator('.payment__errors:has-text("Failed to challenge captcha, please try again later.")').waitFor().then(async () => {
+          console.error('  Failed to challenge captcha, please try again later.');
+          await notify('epic-games: failed to challenge captcha. Please check.');
+        }).catch(_ => { });
+        await page.locator('text=Thanks for your order!').waitFor({ state: 'attached' }); // TODO Bundle: got stuck here
         db.data[user][game_id].status = 'claimed';
         db.data[user][game_id].time = datetime(); // claimed time overwrites failed/dryrun time
         console.log('  Claimed successfully!');

gog.js

@@ -10,13 +10,18 @@ console.log(datetime(), 'started checking gog');
 
 const db = await jsonDb('gog.json', {});
 
+if (cfg.width < 1280) { // otherwise 'Sign in' and #menuUsername are hidden (but attached to DOM), see https://github.com/vogler/free-games-claimer/issues/335
+  console.error(`Window width is set to ${cfg.width} but needs to be at least 1280 for GOG!`);
+  process.exit(1);
+}
+
 // https://playwright.dev/docs/auth#multi-factor-authentication
 const context = await firefox.launchPersistentContext(cfg.dir.browser, {
   headless: cfg.headless,
   viewport: { width: cfg.width, height: cfg.height },
   locale: 'en-US', // ignore OS locale to be sure to have english text for locators -> done via /en in URL
   recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
-  recordHar: cfg.record ? { path: `data/record/gog-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
+  recordHar: cfg.record ? { path: `data/record/gog-${filenamify(datetime())}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
   handleSIGINT: false, // have to handle ourselves and call context.close(), otherwise recordings from above won't be saved
 });
 
@@ -93,10 +98,9 @@ try {
   if (!await banner.count()) {
     console.log('Currently no free giveaway!');
   } else {
-    const text = await page.locator('.giveaway-banner__title').innerText();
+    const text = await page.locator('.giveaway__content-header').innerText();
-    const title = text.match(/Claim (.*)/)[1];
+    const title = text.match(/Claim (.*) and don't miss the/)[1];
-    const slug = await banner.getAttribute('href');
+    const url = await banner.locator('a').first().getAttribute('href');
-    const url = `https://gog.com${slug}`;
     console.log(`Current free game: ${title} - ${url}`);
     db.data[user][title] ||= { title, time: datetime(), url };
     if (cfg.dryrun) process.exit(1);

package.json

@@ -17,20 +17,20 @@
   },
   "type": "module",
   "engines": {
-    "node": ">=15"
+    "node": ">=17"
   },
   "dependencies": {
     "chalk": "^5.3.0",
     "cross-env": "^7.0.3",
-    "dotenv": "^16.3.1",
+    "dotenv": "^16.4.5",
     "enquirer": "^2.4.1",
-    "lowdb": "^6.1.1",
+    "lowdb": "^7.0.1",
     "otplib": "^12.0.1",
-    "playwright-firefox": "^1.40.1",
+    "playwright-firefox": "^1.45.0",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
   },
   "devDependencies": {
-    "@stylistic/eslint-plugin-js": "^1.5.1",
+    "@stylistic/eslint-plugin-js": "^2.2.2",
-    "eslint": "^8.56.0"
+    "eslint": "^9.5.0"
   }
 }

prime-gaming.js

@@ -19,7 +19,7 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, {
   viewport: { width: cfg.width, height: cfg.height },
   locale: 'en-US', // ignore OS locale to be sure to have english text for locators
   recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
-  recordHar: cfg.record ? { path: `data/record/pg-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
+  recordHar: cfg.record ? { path: `data/record/pg-${filenamify(datetime())}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
   handleSIGINT: false, // have to handle ourselves and call context.close(), otherwise recordings from above won't be saved
 });
 
@@ -97,16 +97,50 @@ try {
     process.exit(1);
   }
 
-  await page.click('button[data-type="Game"]');
+  const waitUntilStable = async (f, act) => {
+    let v;
+    while (true) {
+      const v2 = await f();
+      console.log('waitUntilStable', v2);
+      if (v == v2) break;
+      v = v2;
+      await act();
+    }
+  };
+  const scrollUntilStable = async f => waitUntilStable(f, async () => {
     await page.keyboard.press('End'); // scroll to bottom to show all games
     await page.waitForLoadState('networkidle'); // wait for all games to be loaded
-  await page.waitForTimeout(2000); // TODO networkidle wasn't enough to load all already collected games
+    await page.waitForTimeout(5000); // TODO networkidle wasn't enough to load all already collected games
+  });
+
+  await page.click('button[data-type="Game"]');
   const games = page.locator('div[data-a-target="offer-list-FGWP_FULL"]');
   await games.waitFor();
+  await scrollUntilStable(() => games.locator('.item-card__action').count());
   console.log('Number of already claimed games (total):', await games.locator('p:has-text("Collected")').count());
   // can't use .all() since the list of elements via locator will change after click while we iterate over it
-  const internal = await games.locator('.item-card__action:has([data-a-target="FGWPOffer"])').elementHandles();
+  const internal = await games.locator('.item-card__action:has(button[data-a-target="FGWPOffer"])').elementHandles();
-  const external = await games.locator('.item-card__action:has([data-a-target="ExternalOfferClaim"])').all();
+  const external = await games.locator('.item-card__action:has(a[data-a-target="FGWPOffer"])').all();
+  // bottom to top: oldest to newest games
+  internal.reverse();
+  external.reverse();
+  const checkTimeLeft = async url => {
+    // console.log('  Checking time left for game:', url);
+    const check = async p => {
+      console.log(' ', await p.locator('.availability-date').innerText());
+      const dueDateOrg = await p.locator('.availability-date .tw-bold').innerText();
+      const dueDate = datetime(new Date(Date.parse(dueDateOrg + ' 17:00')));
+      console.log('  Due date:', dueDate);
+    };
+    if (page.url() == url) {
+      await check(page);
+    } else {
+      const p = await context.newPage();
+      await p.goto(url, { waitUntil: 'domcontentloaded' });
+      await check(p);
+      p.close();
+    }
+  };
   console.log('Number of free unclaimed games (Prime Gaming):', internal.length);
   // claim games in internal store
   for (const card of internal) {
@@ -115,6 +149,7 @@ try {
     const slug = await (await card.$('a')).getAttribute('href');
     const url = 'https://gaming.amazon.com' + slug.split('?')[0];
     console.log('Current free game:', title);
+    if (cfg.pg_timeLeft) await checkTimeLeft(url);
     if (cfg.dryrun) continue;
     if (cfg.interactive && !await confirm()) continue;
     await (await card.$('.tw-button:has-text("Claim")')).click();
@@ -134,6 +169,7 @@ try {
     // await (await card.$('text=Claim')).click(); // goes to URL of game, no need to wait
     external_info.push({ title, url });
   }
+  // external_info = [ { title: 'Fallout 76 (XBOX)', url: 'https://gaming.amazon.com/fallout-76-xbox-fgwp/dp/amzn1.pg.item.9fe17d7b-b6c2-4f58-b494-cc4e79528d0b?ingress=amzn&ref_=SM_Fallout76XBOX_S01_FGWP_CRWN' } ];
   for (const { title, url } of external_info) {
     console.log('Current free game:', title); // , url);
     await page.goto(url, { waitUntil: 'domcontentloaded' });
@@ -141,6 +177,7 @@ try {
     const item_text = await page.innerText('[data-a-target="DescriptionItemDetails"]');
     const store = item_text.toLowerCase().replace(/.* on /, '').slice(0, -1);
     console.log('  External store:', store);
+    if (cfg.pg_timeLeft) await checkTimeLeft(url);
     if (cfg.dryrun) continue;
     if (cfg.interactive && !await confirm()) continue;
     await Promise.any([page.click('[data-a-target="buy-box"] .tw-button:has-text("Get game")'), page.click('[data-a-target="buy-box"] .tw-button:has-text("Claim")'), page.click('.tw-button:has-text("Complete Claim")'), page.waitForSelector('div:has-text("Link game account")'), page.waitForSelector('.thank-you-title:has-text("Success")')]); // waits for navigation
@@ -163,7 +200,8 @@ try {
       const redeem = {
         // 'origin': 'https://www.origin.com/redeem', // TODO still needed or now only via account linking?
         'gog.com': 'https://www.gog.com/redeem',
-        'microsoft games': 'https://redeem.microsoft.com',
+        'microsoft store': 'https://account.microsoft.com/billing/redeem',
+        xbox: 'https://account.microsoft.com/billing/redeem',
         'legacy games': 'https://www.legacygames.com/primedeal',
       };
       if (store in redeem) { // did not work for linked origin: && !await page.locator('div:has-text("Successfully Claimed")').count()
@@ -172,7 +210,9 @@ try {
         if (store == 'legacy games') { // may be different URL like https://legacygames.com/primeday/puzzleoftheyear/
           redeem[store] = await (await page.$('li:has-text("Click here") a')).getAttribute('href'); // full text: Click here to enter your redemption code.
         }
-        console.log('  URL to redeem game:', redeem[store]);
+        let redeem_url = redeem[store];
+        if (store == 'gog.com') redeem_url += '/' + code; // to log and notify, but can't use for goto below (captcha)
+        console.log('  URL to redeem game:', redeem_url);
         db.data[user][title].code = code;
         let redeem_action = 'redeem';
         if (cfg.pg_redeem) { // try to redeem keys on external stores
@@ -218,33 +258,50 @@ try {
                 console.log('  Unknown Response 2 - please report in https://github.com/vogler/free-games-claimer/issues/5');
               }
             }
-          } else if (store == 'microsoft games') {
+          } else if (store == 'microsoft store' || store == 'xbox') {
-            console.error(`  Redeem on ${store} not yet implemented!`);
+            console.error(`  Redeem on ${store} is experimental!`);
+            // await page2.pause();
             if (page2.url().startsWith('https://login.')) {
-              console.error('  Not logged in! Use the browser to login manually.');
+              console.error('  Not logged in! Use the browser to login manually. Waiting for 60s.');
+              await page2.waitForTimeout(60 * 1000);
               redeem_action = 'redeem (login)';
             } else {
-              const r = page2.waitForResponse(r => r.url().startsWith('https://purchase.mp.microsoft.com/'));
+              const iframe = page2.frameLocator('#redeem-iframe');
-              await page2.fill('[name=tokenString]', code);
+              const input = iframe.locator('[name=tokenString]');
+              await input.waitFor();
+              await input.fill(code);
+              const r = page2.waitForResponse(r => r.url().startsWith('https://cart.production.store-web.dynamics.com/v1.0/Redeem/PrepareRedeem'));
               // console.log(await page2.locator('.redeem_code_error').innerText());
               const rt = await (await r).text();
-              console.debug(`  Response: ${rt}`);
               // {"code":"NotFound","data":[],"details":[],"innererror":{"code":"TokenNotFound",...
-              const reason = JSON.parse(rt).code;
+              const j = JSON.parse(rt);
-              if (reason == 'NotFound') {
+              const reason = j?.events?.cart.length && j.events.cart[0]?.data?.reason;
+              if (reason == 'TokenNotFound') {
                 redeem_action = 'redeem (not found)';
                 console.error('  Code was not found!');
-              } else { // TODO find out other responses
+              } else if (j?.productInfos?.length && j.productInfos[0]?.redeemable) {
-                await page2.click('#nextButton');
+                await iframe.locator('button:has-text("Next")').click();
-                redeem_action = 'redeemed?';
+                await iframe.locator('button:has-text("Confirm")').click();
-                console.log('  Redeemed successfully? Please report your Response from above (if it is new) in https://github.com/vogler/free-games-claimer/issues/5');
+                const r = page2.waitForResponse(r => r.url().startsWith('https://cart.production.store-web.dynamics.com/v1.0/Redeem/RedeemToken'));
+                const j = JSON.parse(await (await r).text());
+                if (j?.events?.cart.length && j.events.cart[0]?.data?.reason == 'UserAlreadyOwnsContent') {
+                  redeem_action = 'already redeemed';
+                  console.error('  error: UserAlreadyOwnsContent');
+                } else if (true) { // TODO what's returned on success?
+                  redeem_action = 'redeemed';
                   db.data[user][title].status = 'claimed and redeemed?';
+                  console.log('  Redeemed successfully? Please report if not in https://github.com/vogler/free-games-claimer/issues/5');
+                }
+              } else { // TODO find out other responses
+                redeem_action = 'unknown';
+                console.debug(`  Response: ${rt}`);
+                console.log('  Redeemed successfully? Please report your Response from above (if it is new) in https://github.com/vogler/free-games-claimer/issues/5');
               }
             }
           } else if (store == 'legacy games') {
             await page2.fill('[name=coupon_code]', code);
-            await page2.fill('[name=email]', cfg.pg_email); // TODO option for sep. email?
+            await page2.fill('[name=email]', cfg.lg_email);
-            await page2.fill('[name=email_validate]', cfg.pg_email);
+            await page2.fill('[name=email_validate]', cfg.lg_email);
             await page2.uncheck('[name=newsletter_sub]');
             await page2.click('[type="submit"]');
             try {
@@ -264,7 +321,7 @@ try {
           if (cfg.debug) await page2.pause();
           await page2.close();
         }
-        notify_game.status = `<a href="${redeem[store]}">${redeem_action}</a> ${code} on ${store}`;
+        notify_game.status = `<a href="${redeem_url}">${redeem_action}</a> ${code} on ${store}`;
       } else {
         notify_game.status = `claimed on ${store}`;
         db.data[user][title].status = 'claimed';
@@ -281,8 +338,7 @@ try {
   if (notify_games.length) { // make screenshot of all games if something was claimed
     const p = screenshot(`${filenamify(datetime())}.png`);
     // await page.screenshot({ path: p, fullPage: true }); // fullPage does not make a difference since scroll not on body but on some element
-    await page.keyboard.press('End'); // scroll to bottom to show all games
+    await scrollUntilStable(() => games.locator('.item-card__action').count());
-    await page.waitForTimeout(1000); // wait for fade in animation
     const viewportSize = page.viewportSize(); // current viewport size
     await page.setViewportSize({ ...viewportSize, height: 3000 }); // increase height, otherwise element screenshot is cut off at the top and bottom
     await games.screenshot({ path: p }); // screenshot of all claimed games
@@ -296,17 +352,7 @@ try {
     await loot.waitFor();
 
     process.stdout.write('Loading all DLCs on page...');
-    let n1 = 0;
+    await scrollUntilStable(() => loot.locator('[data-a-target="item-card"]').count())
-    let n2 = 0;
-    do {
-      n1 = n2;
-      n2 = await loot.locator('[data-a-target="item-card"]').count();
-      // console.log(n2);
-      process.stdout.write(` ${n2}`);
-      await page.keyboard.press('End'); // scroll to bottom to show all dlcs
-      await page.waitForLoadState('networkidle'); // did not wait for dlcs to be loaded
-      await page.waitForTimeout(1000);
-    } while (n2 > n1);
 
     console.log('\nNumber of already claimed DLC:', await loot.locator('p:has-text("Collected")').count());
 

src/config.js

@@ -41,12 +41,13 @@ export const cfg = {
   gog_email: process.env.GOG_EMAIL || process.env.EMAIL,
   gog_password: process.env.GOG_PASSWORD || process.env.PASSWORD,
   gog_newsletter: process.env.GOG_NEWSLETTER == '1', // do not unsubscribe from newsletter after claiming a game
+  // auth AliExpress
+  ae_email: process.env.AE_EMAIL || process.env.EMAIL,
+  ae_password: process.env.AE_PASSWORD || process.env.PASSWORD,
   // OTP only via GOG_EMAIL, can't add app...
-  // auth xbox
+  // experimmental
-  xbox_email: process.env.XBOX_EMAIL || process.env.EMAIL,
-  xbox_password: process.env.XBOX_PASSWORD || process.env.PASSWORD,
-  xbox_otpkey: process.env.XBOX_OTPKEY,
-  // experimmental - likely to change
   pg_redeem: process.env.PG_REDEEM == '1', // prime-gaming: redeem keys on external stores
+  lg_email: process.env.LG_EMAIL || process.env.PG_EMAIL || process.env.EMAIL, // prime-gaming: external: legacy-games: email to use for redeeming
   pg_claimdlc: process.env.PG_CLAIMDLC == '1', // prime-gaming: claim in-game content
+  pg_timeLeft: process.env.PG_TIMELEFT == '1', // prime-gaming: list time left to claim
 };

src/util.js

@@ -11,8 +11,8 @@ export const dataDir = s => path.resolve(__dirname, '..', 'data', s);
 export const resolve = (...a) => a.length && a[0] == '0' ? null : path.resolve(...a);
 
 // json database
-import { JSONPreset } from 'lowdb/node';
+import { JSONFilePreset } from 'lowdb/node';
-export const jsonDb = (file, defaultData) => JSONPreset(dataDir(file), defaultData);
+export const jsonDb = (file, defaultData) => JSONFilePreset(dataDir(file), defaultData);
 
 export const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 // date and time as UTC (no timezone offset) in nicely readable and sortable format, e.g., 2022-10-06 12:05:27.313
@@ -116,7 +116,7 @@ export const notify = html => new Promise((resolve, reject) => {
     return resolve();
   }
   // const cmd = `apprise '${cfg.notify}' ${title} -i html -b '${html}'`; // this had problems if e.g. ' was used in arg; could have `npm i shell-escape`, but instead using safer execFile which takes args as array instead of exec which spawned a shell to execute the command
-  const args = [cfg.notify, '-i', 'html', '-b', html];
+  const args = [cfg.notify, '-i', 'html', '-b', `'${html}'`];
   if (cfg.notify_title) args.push(...['-t', cfg.notify_title]);
   if (cfg.debug) console.debug(`apprise ${args.map(a => `'${a}'`).join(' ')}`); // this also doesn't escape, but it's just for info
   execFile('apprise', args, (error, stdout, stderr) => {

unrealengine.js

@@ -25,7 +25,7 @@ const context = await firefox.launchPersistentContext(cfg.dir.browser, {
   // userAgent for firefox: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0
   locale: 'en-US', // ignore OS locale to be sure to have english text for locators
   recordVideo: cfg.record ? { dir: 'data/record/', size: { width: cfg.width, height: cfg.height } } : undefined, // will record a .webm video for each page navigated; without size, video would be scaled down to fit 800x800
-  recordHar: cfg.record ? { path: `data/record/ue-${datetime()}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
+  recordHar: cfg.record ? { path: `data/record/ue-${filenamify(datetime())}.har` } : undefined, // will record a HAR file with network requests and responses; can be imported in Chrome devtools
   handleSIGINT: false, // have to handle ourselves and call context.close(), otherwise recordings from above won't be saved
 });
 

xbox.js

@@ -1,251 +0,0 @@
-import { firefox } from 'playwright-firefox'; // stealth plugin needs no outdated playwright-extra
-import { authenticator } from 'otplib';
-import {
-  datetime,
-  handleSIGINT,
-  html_game_list,
-  jsonDb,
-  notify,
-  prompt,
-} from './src/util.js';
-import { cfg } from './src/config.js';
-
-// ### SETUP
-const URL_CLAIM = 'https://www.xbox.com/en-US/live/gold'; // #gameswithgold";
-
-console.log(datetime(), 'started checking xbox');
-
-const db = await jsonDb('xbox.json');
-db.data ||= {};
-
-handleSIGINT();
-
-// https://playwright.dev/docs/auth#multi-factor-authentication
-const context = await firefox.launchPersistentContext(cfg.dir.browser, {
-  headless: cfg.headless,
-  viewport: { width: cfg.width, height: cfg.height },
-  locale: 'en-US', // ignore OS locale to be sure to have english text for locators -> done via /en in URL
-});
-
-if (!cfg.debug) context.setDefaultTimeout(cfg.timeout);
-
-const page = context.pages().length
-  ? context.pages()[0]
-  : await context.newPage(); // should always exist
-await page.setViewportSize({ width: cfg.width, height: cfg.height }); // TODO workaround for https://github.com/vogler/free-games-claimer/issues/277 until Playwright fixes it
-
-const notify_games = [];
-let user;
-
-main();
-
-async function main() {
-  try {
-    await performLogin();
-    await getAndSaveUser();
-    await redeemFreeGames();
-  } catch (error) {
-    console.error(error);
-    process.exitCode ||= 1;
-    if (error.message && process.exitCode != 130) notify(`xbox failed: ${error.message.split('\n')[0]}`);
-  } finally {
-    await db.write(); // write out json db
-    if (notify_games.filter(g => g.status != 'existed').length) {
-      // don't notify if all were already claimed
-      notify(`xbox (${user}):<br>${html_game_list(notify_games)}`);
-    }
-    await context.close();
-  }
-}
-
-async function performLogin() {
-  await page.goto(URL_CLAIM, { waitUntil: 'domcontentloaded' }); // default 'load' takes forever
-
-  const signInLocator = page
-    .getByRole('link', {
-      name: 'Sign in to your account',
-    })
-    .first();
-  const usernameLocator = page
-    .getByRole('button', {
-      name: 'Account manager for',
-    })
-    .first();
-
-  await Promise.any([signInLocator.waitFor(), usernameLocator.waitFor()]);
-
-  if (await usernameLocator.isVisible()) {
-    return; // logged in using saved cookie
-  } else if (await signInLocator.isVisible()) {
-    console.error('Not signed in anymore.');
-    await signInLocator.click();
-    await signInToXbox();
-  } else {
-    console.error('lost! where am i?');
-  }
-}
-
-async function signInToXbox() {
-  page.waitForLoadState('domcontentloaded');
-  if (!cfg.debug) context.setDefaultTimeout(cfg.login_timeout); // give user some extra time to log in
-  console.info(`Login timeout is ${cfg.login_timeout / 1000} seconds!`);
-
-  // ### FETCH EMAIL/PASS
-  if (cfg.xbox_email && cfg.xbox_password) console.info('Using email and password from environment.');
-  else console.info(
-    'Press ESC to skip the prompts if you want to login in the browser (not possible in headless mode).',
-  );
-  const email = cfg.xbox_email || await prompt({ message: 'Enter email' });
-  const password =
-        email &&
-        (cfg.xbox_password ||
-            await prompt({
-              type: 'password',
-              message: 'Enter password',
-            }));
-    // ### FILL IN EMAIL/PASS
-  if (email && password) {
-    const usernameLocator = page
-      .getByPlaceholder('Email, phone, or Skype')
-      .first();
-    const passwordLocator = page.getByPlaceholder('Password').first();
-
-    await Promise.any([
-      usernameLocator.waitFor(),
-      passwordLocator.waitFor(),
-    ]);
-
-    // username may already be saved from before, if so, skip to filling in password
-    if (await page.getByPlaceholder('Email, phone, or Skype').isVisible()) {
-      await usernameLocator.fill(email);
-      await page.getByRole('button', { name: 'Next' }).click();
-    }
-
-    await passwordLocator.fill(password);
-    await page.getByRole('button', { name: 'Sign in' }).click();
-
-    // handle MFA, but don't await it
-    page.locator('input[name="otc"]')
-      .waitFor()
-      .then(async () => {
-        console.log('Two-Step Verification - Enter security code');
-        console.log(
-          await page
-            .locator('div[data-bind="text: description"]')
-            .innerText(),
-        );
-        const otp =
-                    cfg.xbox_otpkey &&
-                        authenticator.generate(cfg.xbox_otpkey) ||
-                    await prompt({
-                      type: 'text',
-                      message: 'Enter two-factor sign in code',
-                      validate: n => n.toString().length == 6 ||
-                            'The code must be 6 digits!',
-                    }); // can't use type: 'number' since it strips away leading zeros and codes sometimes have them
-        await page.type('input[name="otc"]', otp.toString());
-        await page
-          .getByLabel('Don\'t ask me again on this device')
-          .check(); // Trust this Browser
-        await page.getByRole('button', { name: 'Verify' }).click();
-      })
-      .catch(_ => {});
-
-    // Trust this browser, but don't await it
-    page.getByLabel('Don\'t show this again')
-      .waitFor()
-      .then(async () => {
-        await page.getByLabel('Don\'t show this again').check();
-        await page.getByRole('button', { name: 'Yes' }).click();
-      })
-      .catch(_ => {});
-  } else {
-    console.log('Waiting for you to login in the browser.');
-    await notify(
-      'xbox: no longer signed in and not enough options set for automatic login.',
-    );
-    if (cfg.headless) {
-      console.log(
-        'Run `SHOW=1 node xbox` to login in the opened browser.',
-      );
-      await context.close();
-      process.exit(1);
-    }
-  }
-
-  // ### VERIFY SIGNED IN
-  await page.waitForURL(`${URL_CLAIM}**`);
-
-  if (!cfg.debug) context.setDefaultTimeout(cfg.timeout);
-}
-
-async function getAndSaveUser() {
-  user = await page.locator('#mectrl_currentAccount_primary').innerHTML();
-  console.log(`Signed in as '${user}'`);
-  db.data[user] ||= {};
-}
-
-async function redeemFreeGames() {
-  const monthlyGamesLocator = await page.locator('.f-size-large').all();
-
-  const monthlyGamesPageLinks = await Promise.all(
-    monthlyGamesLocator.map(
-      async el => await el.locator('a').getAttribute('href'),
-    ),
-  );
-  console.log('Free games:', monthlyGamesPageLinks);
-
-  for (const url of monthlyGamesPageLinks) {
-    await page.goto(url);
-
-    const title = await page.locator('h1').first().innerText();
-    const game_id = page.url().split('/').pop();
-    db.data[user][game_id] ||= { title, time: datetime(), url: page.url() }; // this will be set on the initial run only!
-    console.log('Current free game:', title);
-    const notify_game = { title, url, status: 'failed' };
-    notify_games.push(notify_game); // status is updated below
-
-    // SELECTORS
-    const getBtnLocator = page.getByText('GET', { exact: true }).first();
-    const installToLocator = page
-      .getByText('INSTALL TO', { exact: true })
-      .first();
-
-    await Promise.any([
-      getBtnLocator.waitFor(),
-      installToLocator.waitFor(),
-    ]);
-
-    if (await installToLocator.isVisible()) {
-      console.log('  Already in library! Nothing to claim.');
-      notify_game.status = 'existed';
-      db.data[user][game_id].status ||= 'existed'; // does not overwrite claimed or failed
-    } else if (await getBtnLocator.isVisible()) {
-      console.log('  Not in library yet! Click GET.');
-      await getBtnLocator.click();
-
-      // wait for popup
-      await page
-        .locator('iframe[name="purchase-sdk-hosted-iframe"]')
-        .waitFor();
-      const popupLocator = page.frameLocator(
-        '[name=purchase-sdk-hosted-iframe]',
-      );
-
-      const finalGetBtnLocator = popupLocator.getByText('GET');
-      await finalGetBtnLocator.waitFor();
-      await finalGetBtnLocator.click();
-
-      await page.getByText('Thank you for your purchase.').waitFor();
-      notify_game.status = 'claimed';
-      db.data[user][game_id].status = 'claimed';
-      db.data[user][game_id].time = datetime(); // claimed time overwrites failed/dryrun time
-      console.log('  Claimed successfully!');
-    }
-
-    // notify_game.status = db.data[user][game_id].status; // claimed or failed
-
-    // const p = path.resolve(cfg.dir.screenshots, playstation-plus', `${game_id}.png`);
-    // if (!existsSync(p)) await page.screenshot({ path: p, fullPage: false }); // fullPage is quite long...
-  }
-}