feat(amazon): support for TFA, compatibility with amazon.de in 12-2025/01-2026

package.json

@@ -27,6 +27,7 @@
     "luxon": "^3.4.4",
     "node-cron": "^3.0.3",
     "puppeteer": "^23.6.1",
+    "totp-generator": "^2.0.0",
     "winston": "^3.13.0"
   },
   "devDependencies": {
@@ -109,7 +110,7 @@
   "scripts": {
     "postinstall": "npx --yes ignore-dependency-scripts \"ts-node ./scripts/postinstall.ts\"",
     "prestart": "npm run lint",
-    "start": "cd ./bin && dev scrape amazon",
+    "start": "cd ./bin && ./dev scrape amazon",
     "start:debug": "node --inspect=0.0.0.0:9229 ./bin/run scrape all",
     "prebuild": "npm run clean && npm run lint",
     "build": "tsc -b",

src/classes/key-based-otp.class.ts

@@ -0,0 +1,23 @@
+import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface";
+import { TOTP } from "totp-generator"
+import { Logger } from "winston";
+
+export class KeyBasedOtp implements OtpGenerator
+{
+  constructor(private logger: Logger) {
+  }
+
+  async generate(options: OtpOptions): Promise<string> {
+    if (!('otpKey' in options)) {
+      throw new Error('OTP key required');
+    }
+
+    this.logger.debug(`OTP key provided, generating TOTP token`);
+
+    const otp = await TOTP.generate(options.otpKey);
+
+    this.logger.debug(`Generated OTP token: ${otp.otp}`);
+
+    return otp.otp;
+  }
+}

src/classes/literal-otp.class.ts

@@ -0,0 +1,12 @@
+import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface";
+
+export class LiteralOtp implements OtpGenerator
+{
+  async generate(options: OtpOptions): Promise<string> {
+    if (!('otp' in options)) {
+      throw new Error('OTP required');
+    }
+
+    return options.otp;
+  }
+}

src/classes/otp-generator.class.ts

@@ -0,0 +1,24 @@
+import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface";
+
+export class CompositeOtpGenerator implements OtpGenerator
+{
+  constructor(private keyBased: OtpGenerator, private literal: OtpGenerator, private provider: OtpGenerator)
+  {}
+
+  generate(options: OtpOptions): Promise<string> {
+
+    if ('otp' in options) {
+      return this.literal.generate(options);
+    }
+
+    if ('otpKey' in options) {
+      return this.keyBased.generate(options);
+    }
+
+    if ('otpProvider' in options) {
+      throw new Error('Not implemented');
+    }
+
+    throw new Error('Cannot generate OTP');
+  }
+}

src/classes/scrape-command.class.ts

@@ -23,7 +23,7 @@ export abstract class ScrapeCommand<
 > extends BaseCommand<T> {
   private browser: Browser;
 
-  private pupeteerArgs = [
+  private puppeteerArgs = [
     `--window-size=1920,1080`,
     `--no-sandbox`,
     `--disable-setuid-sandbox`,
@@ -81,7 +81,7 @@ export abstract class ScrapeCommand<
     this.logger.debug(`processJsonFile: ${this.processJsonFile}`);
     this.logger.debug(`fileDestinationFolder: ${this.flags.fileDestinationFolder}`);
     this.logger.debug(`Running in Docker: ${isRunningInDocker()}`);
-    this.browser = await new Puppeteer(this.flags.debug, this.pupeteerArgs, isRunningInDocker()).setup();
+    this.browser = await new Puppeteer(this.flags.debug, this.puppeteerArgs, isRunningInDocker()).setup();
   }
 
   protected async initFlags() {

src/commands/scrape/amazon/helpers/auth.helper.ts

@@ -2,14 +2,18 @@ import { Command } from "@oclif/core";
 import { AmazonSelectors } from "../../../../interfaces/selectors.interface";
 import { Page } from "../../../../classes/puppeteer.class";
 import { AmazonDefinition } from "../../../../interfaces/amazon.interface";
+import { AmazonOptions } from "../../../../interfaces/amazon-options.interface";
+import winston from "winston";
+import { OtpGenerator } from "../../../../interfaces/otp.interface";
 
 export const login = async (
   page: Page,
   selectors: AmazonSelectors,
-  options,
+  options: AmazonOptions,
   amazonUrls: AmazonDefinition,
-  logger,
-  command: Command
+  logger: winston.Logger,
+  command: Command,
+  otp: OtpGenerator
 ): Promise<boolean> => {
   let hasMessages = false;
 
@@ -36,14 +40,9 @@ export const login = async (
     }
   };
 
-  while (!hasMessages) {
-    if (!options.username && !options.password) {
-      // options.username = await ux.action(`What is your amazaon username?`);
-      // options.password = await ux.prompt(`What is your password?`, {
-      //   type: `hide`,
-      // });
-    }
+  await deactivatePasskeys(page);
 
+  while (!hasMessages) {
     logger.debug(`Selectors: ${JSON.stringify(selectors, null, 4)}`);
 
     await page.goto(amazonUrls.loginPage);
@@ -52,10 +51,14 @@ export const login = async (
     await page.click(`input[type=submit]`);
     await page.waitForNavigation();
 
+    logger.debug('Username successfully entered')
+
     await page.type(`input[type=password]`, options.password);
     await page.click(`input[type=submit]`);
     await page.waitForNavigation();
 
+    logger.debug('Password successfully entered')
+
     const authErrors = await checkForAuthMessages(`Error`);
     const authWarning = await checkForAuthMessages(`Warning`);
 
@@ -78,10 +81,9 @@ export const login = async (
 
     if (page.url().indexOf(`/mfa?`) > -1) {
       logger.info(`MFA detected`);
-      // const secondFactor = await ux.prompt(`What is your two-factor token?`, {
-      //   type: `mask`,
-      // });
-      // await page.type(`input#auth-mfa-otpcode`, secondFactor);
+      const secondFactor = await otp.generate(options)
+
+      await page.type(`input#auth-mfa-otpcode`, secondFactor);
       await page.click(`input#auth-mfa-remember-device`);
       await page.click(`input[type=submit]`);
 
@@ -91,3 +93,18 @@ export const login = async (
     return true;
   }
 };
+
+async function deactivatePasskeys(page: Page) {
+  const cdp = await page.createCDPSession();
+  await cdp.send(`WebAuthn.enable`);
+  await cdp.send(`WebAuthn.addVirtualAuthenticator`, {
+    options: {
+      protocol: `ctap2`,
+      transport: `internal`,
+      hasResidentKey: true,
+      hasUserVerification: true,
+      isUserVerified: true,
+      automaticPresenceSimulation: true,
+    }
+  });
+}

src/commands/scrape/amazon/helpers/selectors.helper.ts

@@ -1,15 +1,15 @@
 import { AmazonSelectors } from "../../../../interfaces/selectors.interface";
 
 export const amazonSelectors: AmazonSelectors = {
-  orderCards: `div.order.js-order-card`,
-  invoiceSpans: `span.hide-if-no-js .a-declarative[data-action="a-popover"]`,
-  orderNr: `.yohtmlc-order-id span:nth-last-child(1) bdi`,
+  orderCards: `.order-card`,
+  invoiceSpans: `.yohtmlc-order-level-connections [data-action="a-popover"]`,
+  orderNr: `.yohtmlc-order-id span:nth-last-child(1)`,
   orderDate: `.a-column .a-row:nth-last-child(1) span`,
-  popover: `#a-popover-content-{{index}}`,
+  popover: `.a-popover`,
   invoiceList: `ul.invoice-list`,
   invoiceLinks: `a[href*="invoice.pdf"]`,
   pagination: `ul.a-pagination li.a-normal:nth-last-child(2) a`,
-  yearFilter: `select[name='orderFilter']#orderFilter`,
+  yearFilter: `select[name='timeFilter']#time-filter`,
   authError: `#auth-error-message-box .a-unordered-list li`,
   authWarning: `#auth-warning-message-box .a-unordered-list li`,
   // captchaImage: `div.cvf-captcha-img img[alt~="captcha"]`,

src/commands/scrape/amazon/index.ts

@@ -13,6 +13,9 @@ import { Scrape } from "../../../interfaces/scrape.interface";
 import { AmazonSelectors } from "../../../interfaces/selectors.interface";
 import { WebsiteRun } from "../../../interfaces/website-run.interface";
 import { amazonSelectors } from "./helpers/selectors.helper";
+import { CompositeOtpGenerator } from "../../../classes/otp-generator.class";
+import { KeyBasedOtp } from "../../../classes/key-based-otp.class";
+import { LiteralOtp } from "../../../classes/literal-otp.class";
 
 export default class Amazon extends ScrapeCommand<typeof Amazon> {
   public pluginName = `amazon`;
@@ -26,7 +29,10 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
   static flags = {
     username: Flags.string({ char: `u`, description: `Username`, required: true, env: `AMAZON_USERNAME` }),
     password: Flags.string({ char: `p`, description: `Password`, required: true, env: `AMAZON_PASSWORD` }),
-    tld: Flags.string({ char: `t`, description: `Amazon top level domain`, default: `de`, env: `AMAZON_TLD` }),
+    otpKey: Flags.string({ description: "OTP key to derive TOTP codes from", env: `AMAZON_OTP_KEY`, exactlyOne: ['otp', 'otpProvider'], parse: async v => v.replace(/\s/g, '') }, ),
+    otp: Flags.string({ description: "OTP value", env: `AMAZON_OTP`, exactlyOne: ['otpKey', 'otpProvider']}, ),
+    otpProvider: Flags.url({ description: "OTP provider URL to fetch, e.g. vault://localhost:123/totp/code/my-key", env: `AMAZON_OTP_PROVIDER`, exactlyOne: ['otpKey', 'otp']}, ),
+    tld: Flags.string({ char: `t`, description: `Amazon top level domain`, default: `de`, env: `AMAZON_TLD`}),
     yearFilter: Flags.integer({ aliases: [`yearFilter`], description: `Filters a year`, env: `AMAZON_YEAR_FILTER` }),
     pageFilter: Flags.integer({ aliases: [`pageFilter`], description: `Filters a page`, env: `AMAZON_PAGE_FILTER` }),
   };
@@ -50,7 +56,9 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
     this.selectors = amazonSelectors;
     this.definition = amazon;
 
-    const loggedIn = await login(this.currentPage, amazonSelectors, this.options, amazon, this.logger, this);
+    const otp = new CompositeOtpGenerator(new KeyBasedOtp(this.logger), new LiteralOtp(), null!);
+
+    const loggedIn = await login(this.currentPage, amazonSelectors, this.options, amazon, this.logger, this, otp);
     let processedOrders: Scrape[] = [];
 
     if (loggedIn) {
@@ -103,7 +111,7 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
 
   private async goToOrderPage(amazon: AmazonDefinition): Promise<HTTPResponse> {
     this.logger.debug(`Going to order page...`);
-    return await this.goToYearAndPage(DateTime.now().year, 0, amazon);
+    return await this.goToYearAndPage(DateTime.now().year, 1, amazon);
   }
 
   private async processYears(): Promise<Scrape[]> {
@@ -147,17 +155,21 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
 
     let onlyNewInvoiceHandled = false;
 
-    for (const [orderIndex, orderCard] of orderCards.entries()) {
+    for (const orderCard of orderCards) {
       const { orderNumber, order } = await this.getOrder(orderCard);
-      await this.clickInvoiceSpan(orderCard, orderIndex);
-      const invoiceUrls = await this.getInvoiceUrls(orderIndex);
+      // Find existing popovers *before* clicking the next invoice span to detect which popover is new
+      const existingPopoverIds = await this.findExistingPopoverIds()
+      if (!await this.clickInvoiceSpan(orderCard)) {
+        continue;
+      }
+      const invoiceUrls = await this.getInvoiceUrls(existingPopoverIds);
 
       if (this.options.onlyNew && (orderNumber == this.lastScrapeWithInvoices?.number)) {
         this.logger.info(`Order ${orderNumber} already handled. Exiting.`);
         onlyNewInvoiceHandled = true;
       }
 
-      order.invoices = this.getInvoices(invoiceUrls, orderIndex);
+      order.invoices = this.getInvoices(invoiceUrls, orderNumber);
       processedOrders.push(order);
       this.logger.info(`Processing "${processedOrders.length}" orders`);
 
@@ -171,10 +183,15 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
     return onlyNewInvoiceHandled;
   }
 
+  private async findExistingPopoverIds(): Promise<string[]>
+  {
+    return await this.currentPage.$$eval('.a-popover', (popovers: Array<HTMLDivElement>) => popovers.map(e => e.id))
+  }
+
   private async getOrderPageCount(year: number): Promise<number> {
     this.logger.debug(`Determining order pages...`);
     let orderPageCount: number = null;
-    await this.goToYearAndPage(year, 0, this.definition);
+    await this.goToYearAndPage(year, 1, this.definition);
 
     try {
       orderPageCount = await (await this.currentPage.waitForSelector(this.selectors.pagination, { timeout: this.selectorWaitTimeout })).evaluate((handle: HTMLElement) => parseInt(handle.innerText));
@@ -189,25 +206,31 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
 
   private async goToYearAndPage(year: number, orderPage: number, amazon: AmazonDefinition): Promise<HTTPResponse> {
     this.logger.debug(`Going to year... ${year} order page ${orderPage}`);
-    const nextPageUrl = new URL(`?ie=UTF8&orderFilter=year-${year}&search=&startIndex=${10 * (orderPage)}`, amazon.orderPage);
+    const nextPageUrl = new URL(`?ie=UTF8&timeFilter=year-${year}&search=&startIndex=${10 * (orderPage - 1)}`, amazon.orderPage);
     return await this.currentPage.goto(nextPageUrl.toString());
   }
 
-  private async clickInvoiceSpan(orderCard: ElementHandle<Element>, orderIndex: number): Promise<void> {
+  private async clickInvoiceSpan(orderCard: ElementHandle<Element>): Promise<boolean> {
     const invoiceSpan = await orderCard.$(this.selectors.invoiceSpans);
-    invoiceSpan.click();
-    this.logger.debug(`Checking popover ${orderIndex + 1}`);
+
+    if (invoiceSpan === null) {
+      return false;
+    }
+
+    await invoiceSpan.click();
+    this.logger.debug(`Checking popover`);
+    return true;
   }
 
-  private getInvoices(invoiceUrls: string[], orderIndex: number): Invoice[] {
+  private getInvoices(invoiceUrls: string[], orderNumber: string): Invoice[] {
     this.logger.debug(`Getting invoices...`);
     if (invoiceUrls.length == 0) {
       this.logger.warn(`No invoices found. Order may be undelivered. Check again later.`);
       return [];
     } else {
       const invoices: Array<Invoice> = invoiceUrls.map(invoiceUrl => ({ url: invoiceUrl, status: InvoiceStatus.determined } as Invoice));
       this.logger.info(`${invoices.length} invoices found 📃`);
-      this.logger.debug(`Got invoice url ${(orderIndex + 1)} -> ${invoiceUrls}`);
+      this.logger.debug(`Got invoice URL for ${orderNumber} -> ${invoiceUrls}`);
       return invoices;
     }
   }
@@ -263,17 +286,23 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
     return possibleYears;
   }
 
-  private async getInvoiceUrls(orderIndex: number): Promise<string[]> {
+  private async getInvoiceUrls(previouslyExistingPopoverIds: string[]): Promise<string[]> {
     this.logger.debug(`Getting invoice urls...`);
     let invoiceUrls: string[] = [];
-    const popoverSelectorResolved = this.selectors.popover.replace(`{{index}}`, (orderIndex + 1).toString());
+
+    let popoverSelectorResolved = this.selectors.popover
+    if (previouslyExistingPopoverIds.length > 0) {
+      popoverSelectorResolved = `${this.selectors.popover}:not(${previouslyExistingPopoverIds.map(id => `#${id}`).join(',')})`;
+    }
+
+    this.logger.debug(`Popover selector ${popoverSelectorResolved}`)
 
     try {
       const popover = await this.currentPage.waitForSelector(popoverSelectorResolved, { timeout: this.selectorWaitTimeout });
-      this.logger.debug(`Got popover ${(orderIndex + 1)} -> ${popover}`);
+      this.logger.debug(`Got popover -> ${popover}`);
       const invoiceList = await popover.waitForSelector(this.selectors.invoiceList, { timeout: this.selectorWaitTimeout });
       invoiceUrls = await invoiceList.$$eval(this.selectors.invoiceLinks, (handles: HTMLAnchorElement[]) => handles.map(a => a.href));
-      this.logger.debug(`Got invoiceUrls ${(orderIndex + 1)} -> ${invoiceUrls}`);
+      this.logger.debug(`Got invoiceUrls -> ${invoiceUrls}`);
     } catch (ex) {
       this.logger.error(`Couldn't get popover ${popoverSelectorResolved} within ${this.selectorWaitTimeout}ms. Skipping. ${ex.message}`);
     }