diff --git a/bin/dev b/bin/dev old mode 100644 new mode 100755 diff --git a/bin/run b/bin/run old mode 100644 new mode 100755 diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 6452dedb..2b4bedc5 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -17,6 +17,7 @@ "luxon": "^3.4.4", "node-cron": "^3.0.3", "puppeteer": "^23.6.1", + "totp-generator": "^2.0.0", "winston": "^3.13.0" }, "bin": { @@ -12992,6 +12993,15 @@ "node": "*" } }, + "node_modules/jssha": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.1.tgz", + "integrity": "sha512-VCMZj12FCFMQYcFLPRm/0lOBbLi8uM2BhXPTqw3U4YAfs4AZfiApOoBLoN8cQE60Z50m1MYMTQVCfgF/KaCVhQ==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -19930,6 +19940,15 @@ "node": ">=8.0" } }, + "node_modules/totp-generator": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/totp-generator/-/totp-generator-2.0.0.tgz", + "integrity": "sha512-YXqrJupB/w762T4PrI9qLg5ekb0Of1MRerIW5wh3GRRkH/mgSROw5Gale0gidtc4CfTsNNyZFStS7H4uXJgL2Q==", + "license": "MIT", + "dependencies": { + "jssha": "^3.3.1" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", diff --git a/package.json b/package.json index 440fb964..cc8807f7 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "luxon": "^3.4.4", "node-cron": "^3.0.3", "puppeteer": "^23.6.1", + "totp-generator": "^2.0.0", "winston": "^3.13.0" }, "devDependencies": { @@ -109,7 +110,7 @@ "scripts": { "postinstall": "npx --yes ignore-dependency-scripts \"ts-node ./scripts/postinstall.ts\"", "prestart": "npm run lint", - "start": "cd ./bin && dev scrape amazon", + "start": "cd ./bin && ./dev scrape amazon", "start:debug": "node --inspect=0.0.0.0:9229 ./bin/run scrape all", "prebuild": "npm run clean && npm run lint", "build": "tsc -b", diff --git a/src/classes/key-based-otp.class.ts b/src/classes/key-based-otp.class.ts new file mode 100644 index 00000000..271e4a54 --- /dev/null +++ b/src/classes/key-based-otp.class.ts @@ -0,0 +1,23 @@ +import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface"; +import { TOTP } from "totp-generator" +import { Logger } from "winston"; + +export class KeyBasedOtp implements OtpGenerator +{ + constructor(private logger: Logger) { + } + + async generate(options: OtpOptions): Promise { + if (!('otpKey' in options)) { + throw new Error('OTP key required'); + } + + this.logger.debug(`OTP key provided, generating TOTP token`); + + const otp = await TOTP.generate(options.otpKey); + + this.logger.debug(`Generated OTP token: ${otp.otp}`); + + return otp.otp; + } +} diff --git a/src/classes/literal-otp.class.ts b/src/classes/literal-otp.class.ts new file mode 100644 index 00000000..4bbf8bec --- /dev/null +++ b/src/classes/literal-otp.class.ts @@ -0,0 +1,12 @@ +import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface"; + +export class LiteralOtp implements OtpGenerator +{ + async generate(options: OtpOptions): Promise { + if (!('otp' in options)) { + throw new Error('OTP required'); + } + + return options.otp; + } +} diff --git a/src/classes/otp-generator.class.ts b/src/classes/otp-generator.class.ts new file mode 100644 index 00000000..3e4ad290 --- /dev/null +++ b/src/classes/otp-generator.class.ts @@ -0,0 +1,24 @@ +import { OtpGenerator, OtpOptions } from "../interfaces/otp.interface"; + +export class CompositeOtpGenerator implements OtpGenerator +{ + constructor(private keyBased: OtpGenerator, private literal: OtpGenerator, private provider: OtpGenerator) + {} + + generate(options: OtpOptions): Promise { + + if ('otp' in options) { + return this.literal.generate(options); + } + + if ('otpKey' in options) { + return this.keyBased.generate(options); + } + + if ('otpProvider' in options) { + throw new Error('Not implemented'); + } + + throw new Error('Cannot generate OTP'); + } +} diff --git a/src/classes/scrape-command.class.ts b/src/classes/scrape-command.class.ts index 06137f05..d4511d05 100644 --- a/src/classes/scrape-command.class.ts +++ b/src/classes/scrape-command.class.ts @@ -23,7 +23,7 @@ export abstract class ScrapeCommand< > extends BaseCommand { private browser: Browser; - private pupeteerArgs = [ + private puppeteerArgs = [ `--window-size=1920,1080`, `--no-sandbox`, `--disable-setuid-sandbox`, @@ -81,7 +81,7 @@ export abstract class ScrapeCommand< this.logger.debug(`processJsonFile: ${this.processJsonFile}`); this.logger.debug(`fileDestinationFolder: ${this.flags.fileDestinationFolder}`); this.logger.debug(`Running in Docker: ${isRunningInDocker()}`); - this.browser = await new Puppeteer(this.flags.debug, this.pupeteerArgs, isRunningInDocker()).setup(); + this.browser = await new Puppeteer(this.flags.debug, this.puppeteerArgs, isRunningInDocker()).setup(); } protected async initFlags() { diff --git a/src/commands/scrape/amazon/helpers/auth.helper.ts b/src/commands/scrape/amazon/helpers/auth.helper.ts index 3f53756b..401d384b 100644 --- a/src/commands/scrape/amazon/helpers/auth.helper.ts +++ b/src/commands/scrape/amazon/helpers/auth.helper.ts @@ -2,14 +2,18 @@ import { Command } from "@oclif/core"; import { AmazonSelectors } from "../../../../interfaces/selectors.interface"; import { Page } from "../../../../classes/puppeteer.class"; import { AmazonDefinition } from "../../../../interfaces/amazon.interface"; +import { AmazonOptions } from "../../../../interfaces/amazon-options.interface"; +import winston from "winston"; +import { OtpGenerator } from "../../../../interfaces/otp.interface"; export const login = async ( page: Page, selectors: AmazonSelectors, - options, + options: AmazonOptions, amazonUrls: AmazonDefinition, - logger, - command: Command + logger: winston.Logger, + command: Command, + otp: OtpGenerator ): Promise => { let hasMessages = false; @@ -36,14 +40,9 @@ export const login = async ( } }; - while (!hasMessages) { - if (!options.username && !options.password) { - // options.username = await ux.action(`What is your amazaon username?`); - // options.password = await ux.prompt(`What is your password?`, { - // type: `hide`, - // }); - } + await deactivatePasskeys(page); + while (!hasMessages) { logger.debug(`Selectors: ${JSON.stringify(selectors, null, 4)}`); await page.goto(amazonUrls.loginPage); @@ -52,10 +51,14 @@ export const login = async ( await page.click(`input[type=submit]`); await page.waitForNavigation(); + logger.debug('Username successfully entered') + await page.type(`input[type=password]`, options.password); await page.click(`input[type=submit]`); await page.waitForNavigation(); + logger.debug('Password successfully entered') + const authErrors = await checkForAuthMessages(`Error`); const authWarning = await checkForAuthMessages(`Warning`); @@ -78,10 +81,9 @@ export const login = async ( if (page.url().indexOf(`/mfa?`) > -1) { logger.info(`MFA detected`); - // const secondFactor = await ux.prompt(`What is your two-factor token?`, { - // type: `mask`, - // }); - // await page.type(`input#auth-mfa-otpcode`, secondFactor); + const secondFactor = await otp.generate(options) + + await page.type(`input#auth-mfa-otpcode`, secondFactor); await page.click(`input#auth-mfa-remember-device`); await page.click(`input[type=submit]`); @@ -91,3 +93,18 @@ export const login = async ( return true; } }; + +async function deactivatePasskeys(page: Page) { + const cdp = await page.createCDPSession(); + await cdp.send(`WebAuthn.enable`); + await cdp.send(`WebAuthn.addVirtualAuthenticator`, { + options: { + protocol: `ctap2`, + transport: `internal`, + hasResidentKey: true, + hasUserVerification: true, + isUserVerified: true, + automaticPresenceSimulation: true, + } + }); +} diff --git a/src/commands/scrape/amazon/helpers/selectors.helper.ts b/src/commands/scrape/amazon/helpers/selectors.helper.ts index 60e78fa0..0c403f74 100644 --- a/src/commands/scrape/amazon/helpers/selectors.helper.ts +++ b/src/commands/scrape/amazon/helpers/selectors.helper.ts @@ -1,15 +1,15 @@ import { AmazonSelectors } from "../../../../interfaces/selectors.interface"; export const amazonSelectors: AmazonSelectors = { - orderCards: `div.order.js-order-card`, - invoiceSpans: `span.hide-if-no-js .a-declarative[data-action="a-popover"]`, - orderNr: `.yohtmlc-order-id span:nth-last-child(1) bdi`, + orderCards: `.order-card`, + invoiceSpans: `.yohtmlc-order-level-connections [data-action="a-popover"]`, + orderNr: `.yohtmlc-order-id span:nth-last-child(1)`, orderDate: `.a-column .a-row:nth-last-child(1) span`, - popover: `#a-popover-content-{{index}}`, + popover: `.a-popover`, invoiceList: `ul.invoice-list`, invoiceLinks: `a[href*="invoice.pdf"]`, pagination: `ul.a-pagination li.a-normal:nth-last-child(2) a`, - yearFilter: `select[name='orderFilter']#orderFilter`, + yearFilter: `select[name='timeFilter']#time-filter`, authError: `#auth-error-message-box .a-unordered-list li`, authWarning: `#auth-warning-message-box .a-unordered-list li`, // captchaImage: `div.cvf-captcha-img img[alt~="captcha"]`, diff --git a/src/commands/scrape/amazon/index.ts b/src/commands/scrape/amazon/index.ts index aed15013..f259b4b6 100644 --- a/src/commands/scrape/amazon/index.ts +++ b/src/commands/scrape/amazon/index.ts @@ -13,6 +13,9 @@ import { Scrape } from "../../../interfaces/scrape.interface"; import { AmazonSelectors } from "../../../interfaces/selectors.interface"; import { WebsiteRun } from "../../../interfaces/website-run.interface"; import { amazonSelectors } from "./helpers/selectors.helper"; +import { CompositeOtpGenerator } from "../../../classes/otp-generator.class"; +import { KeyBasedOtp } from "../../../classes/key-based-otp.class"; +import { LiteralOtp } from "../../../classes/literal-otp.class"; export default class Amazon extends ScrapeCommand { public pluginName = `amazon`; @@ -26,7 +29,10 @@ export default class Amazon extends ScrapeCommand { static flags = { username: Flags.string({ char: `u`, description: `Username`, required: true, env: `AMAZON_USERNAME` }), password: Flags.string({ char: `p`, description: `Password`, required: true, env: `AMAZON_PASSWORD` }), - tld: Flags.string({ char: `t`, description: `Amazon top level domain`, default: `de`, env: `AMAZON_TLD` }), + otpKey: Flags.string({ description: "OTP key to derive TOTP codes from", env: `AMAZON_OTP_KEY`, exactlyOne: ['otp', 'otpProvider'], parse: async v => v.replace(/\s/g, '') }, ), + otp: Flags.string({ description: "OTP value", env: `AMAZON_OTP`, exactlyOne: ['otpKey', 'otpProvider']}, ), + otpProvider: Flags.url({ description: "OTP provider URL to fetch, e.g. vault://localhost:123/totp/code/my-key", env: `AMAZON_OTP_PROVIDER`, exactlyOne: ['otpKey', 'otp']}, ), + tld: Flags.string({ char: `t`, description: `Amazon top level domain`, default: `de`, env: `AMAZON_TLD`}), yearFilter: Flags.integer({ aliases: [`yearFilter`], description: `Filters a year`, env: `AMAZON_YEAR_FILTER` }), pageFilter: Flags.integer({ aliases: [`pageFilter`], description: `Filters a page`, env: `AMAZON_PAGE_FILTER` }), }; @@ -50,7 +56,9 @@ export default class Amazon extends ScrapeCommand { this.selectors = amazonSelectors; this.definition = amazon; - const loggedIn = await login(this.currentPage, amazonSelectors, this.options, amazon, this.logger, this); + const otp = new CompositeOtpGenerator(new KeyBasedOtp(this.logger), new LiteralOtp(), null!); + + const loggedIn = await login(this.currentPage, amazonSelectors, this.options, amazon, this.logger, this, otp); let processedOrders: Scrape[] = []; if (loggedIn) { @@ -103,7 +111,7 @@ export default class Amazon extends ScrapeCommand { private async goToOrderPage(amazon: AmazonDefinition): Promise { this.logger.debug(`Going to order page...`); - return await this.goToYearAndPage(DateTime.now().year, 0, amazon); + return await this.goToYearAndPage(DateTime.now().year, 1, amazon); } private async processYears(): Promise { @@ -147,17 +155,21 @@ export default class Amazon extends ScrapeCommand { let onlyNewInvoiceHandled = false; - for (const [orderIndex, orderCard] of orderCards.entries()) { + for (const orderCard of orderCards) { const { orderNumber, order } = await this.getOrder(orderCard); - await this.clickInvoiceSpan(orderCard, orderIndex); - const invoiceUrls = await this.getInvoiceUrls(orderIndex); + // Find existing popovers *before* clicking the next invoice span to detect which popover is new + const existingPopoverIds = await this.findExistingPopoverIds() + if (!await this.clickInvoiceSpan(orderCard)) { + continue; + } + const invoiceUrls = await this.getInvoiceUrls(existingPopoverIds); if (this.options.onlyNew && (orderNumber == this.lastScrapeWithInvoices?.number)) { this.logger.info(`Order ${orderNumber} already handled. Exiting.`); onlyNewInvoiceHandled = true; } - order.invoices = this.getInvoices(invoiceUrls, orderIndex); + order.invoices = this.getInvoices(invoiceUrls, orderNumber); processedOrders.push(order); this.logger.info(`Processing "${processedOrders.length}" orders`); @@ -171,10 +183,15 @@ export default class Amazon extends ScrapeCommand { return onlyNewInvoiceHandled; } + private async findExistingPopoverIds(): Promise + { + return await this.currentPage.$$eval('.a-popover', (popovers: Array) => popovers.map(e => e.id)) + } + private async getOrderPageCount(year: number): Promise { this.logger.debug(`Determining order pages...`); let orderPageCount: number = null; - await this.goToYearAndPage(year, 0, this.definition); + await this.goToYearAndPage(year, 1, this.definition); try { orderPageCount = await (await this.currentPage.waitForSelector(this.selectors.pagination, { timeout: this.selectorWaitTimeout })).evaluate((handle: HTMLElement) => parseInt(handle.innerText)); @@ -189,17 +206,23 @@ export default class Amazon extends ScrapeCommand { private async goToYearAndPage(year: number, orderPage: number, amazon: AmazonDefinition): Promise { this.logger.debug(`Going to year... ${year} order page ${orderPage}`); - const nextPageUrl = new URL(`?ie=UTF8&orderFilter=year-${year}&search=&startIndex=${10 * (orderPage)}`, amazon.orderPage); + const nextPageUrl = new URL(`?ie=UTF8&timeFilter=year-${year}&search=&startIndex=${10 * (orderPage - 1)}`, amazon.orderPage); return await this.currentPage.goto(nextPageUrl.toString()); } - private async clickInvoiceSpan(orderCard: ElementHandle, orderIndex: number): Promise { + private async clickInvoiceSpan(orderCard: ElementHandle): Promise { const invoiceSpan = await orderCard.$(this.selectors.invoiceSpans); - invoiceSpan.click(); - this.logger.debug(`Checking popover ${orderIndex + 1}`); + + if (invoiceSpan === null) { + return false; + } + + await invoiceSpan.click(); + this.logger.debug(`Checking popover`); + return true; } - private getInvoices(invoiceUrls: string[], orderIndex: number): Invoice[] { + private getInvoices(invoiceUrls: string[], orderNumber: string): Invoice[] { this.logger.debug(`Getting invoices...`); if (invoiceUrls.length == 0) { this.logger.warn(`No invoices found. Order may be undelivered. Check again later.`); @@ -207,7 +230,7 @@ export default class Amazon extends ScrapeCommand { } else { const invoices: Array = invoiceUrls.map(invoiceUrl => ({ url: invoiceUrl, status: InvoiceStatus.determined } as Invoice)); this.logger.info(`${invoices.length} invoices found 📃`); - this.logger.debug(`Got invoice url ${(orderIndex + 1)} -> ${invoiceUrls}`); + this.logger.debug(`Got invoice URL for ${orderNumber} -> ${invoiceUrls}`); return invoices; } } @@ -263,17 +286,23 @@ export default class Amazon extends ScrapeCommand { return possibleYears; } - private async getInvoiceUrls(orderIndex: number): Promise { + private async getInvoiceUrls(previouslyExistingPopoverIds: string[]): Promise { this.logger.debug(`Getting invoice urls...`); let invoiceUrls: string[] = []; - const popoverSelectorResolved = this.selectors.popover.replace(`{{index}}`, (orderIndex + 1).toString()); + + let popoverSelectorResolved = this.selectors.popover + if (previouslyExistingPopoverIds.length > 0) { + popoverSelectorResolved = `${this.selectors.popover}:not(${previouslyExistingPopoverIds.map(id => `#${id}`).join(',')})`; + } + + this.logger.debug(`Popover selector ${popoverSelectorResolved}`) try { const popover = await this.currentPage.waitForSelector(popoverSelectorResolved, { timeout: this.selectorWaitTimeout }); - this.logger.debug(`Got popover ${(orderIndex + 1)} -> ${popover}`); + this.logger.debug(`Got popover -> ${popover}`); const invoiceList = await popover.waitForSelector(this.selectors.invoiceList, { timeout: this.selectorWaitTimeout }); invoiceUrls = await invoiceList.$$eval(this.selectors.invoiceLinks, (handles: HTMLAnchorElement[]) => handles.map(a => a.href)); - this.logger.debug(`Got invoiceUrls ${(orderIndex + 1)} -> ${invoiceUrls}`); + this.logger.debug(`Got invoiceUrls -> ${invoiceUrls}`); } catch (ex) { this.logger.error(`Couldn't get popover ${popoverSelectorResolved} within ${this.selectorWaitTimeout}ms. Skipping. ${ex.message}`); } diff --git a/src/interfaces/amazon-options.interface.ts b/src/interfaces/amazon-options.interface.ts index 52794589..a40934cb 100644 --- a/src/interfaces/amazon-options.interface.ts +++ b/src/interfaces/amazon-options.interface.ts @@ -1,6 +1,7 @@ import { LogLevel } from "../enums/loglevel"; +import { OtpOptions } from "./otp.interface"; -export interface AmazonOptions { +export type AmazonOptions = { logLevel: LogLevel, debug: boolean, logPath: string, @@ -15,4 +16,4 @@ export interface AmazonOptions { pageFilter: number, onlyNew: boolean, subFolderForPages: boolean, -} \ No newline at end of file +} & OtpOptions diff --git a/src/interfaces/otp.interface.ts b/src/interfaces/otp.interface.ts new file mode 100644 index 00000000..24676b98 --- /dev/null +++ b/src/interfaces/otp.interface.ts @@ -0,0 +1,7 @@ +export type OtpOptions = { otpKey: string } + | { otp: string } + | { otpProvider: URL} + +export interface OtpGenerator { + generate(options: OtpOptions): Promise +}