Add Image Captcha Solver (#1612)

* First pass - image captcha

* Add SVG converter

* Feature complete

* Add docs

* Types

* Fix up types

* Fix up linting issues

* Lint fixes

* Minor clean-up

* Add tests

* Lint fix

* Fix netlify

* Revert netlify fix

* Code Review Feedback

* Add type checks for input

* fix: exports issue

* Fix else statement

* Group success conditions

---------

Co-authored-by: Alex <[email protected]>

Author: brianhall

injected/integration-test/broker-protection-tests/broker-protection-captcha.spec.js

@@ -1,6 +1,11 @@
 import { test as base } from '@playwright/test';
 import { createConfiguredDbpTest } from './fixtures';
-import { createGetRecaptchaInfoAction, createSolveRecaptchaAction } from '../mocks/broker-protection/captcha.js';
+import {
+    createGetRecaptchaInfoAction,
+    createSolveRecaptchaAction,
+    createGetImageCaptchaInfoAction,
+    createSolveImageCaptchaAction,
+} from '../mocks/broker-protection/captcha.js';
 import { BROKER_PROTECTION_CONFIGS } from './tests-config.js';
 
 const test = createConfiguredDbpTest(base);
@@ -99,4 +104,38 @@ test.describe('Broker Protection Captcha', () => {
             dbp.isQueryParamRemoved(sucessResponse);
         });
     });
+
+    test.describe('image captcha', () => {
+        const imageCaptchaTargetPage = 'image-captcha.html';
+        const imageCaptchaResponseSelector = '#svgCaptchaInputId';
+
+        test.describe('getCaptchaInfo', () => {
+            test('returns the expected response for the correct action data', async ({ createConfiguredDbp }) => {
+                const dbp = await createConfiguredDbp(BROKER_PROTECTION_CONFIGS.default);
+                await dbp.navigatesTo(imageCaptchaTargetPage);
+                await dbp.receivesInlineAction(createGetImageCaptchaInfoAction({ selector: '#svg-captcha-rendering svg' }));
+                const sucessResponse = await dbp.getSuccessResponse();
+                dbp.isCaptchaMatch(sucessResponse, { captchaType: 'image', targetPage: imageCaptchaTargetPage });
+            });
+
+            test('returns an error response when the selector is not an svg or image tag', async ({ createConfiguredDbp }) => {
+                const dbp = await createConfiguredDbp(BROKER_PROTECTION_CONFIGS.default);
+                await dbp.navigatesTo(imageCaptchaTargetPage);
+                await dbp.receivesInlineAction(createGetRecaptchaInfoAction({ selector: '#svg-captcha-rendering' }));
+
+                await dbp.isCaptchaError();
+            });
+        });
+
+        test.describe('solveCaptchaInfo', () => {
+            test('solves the captcha for the correct action data', async ({ createConfiguredDbp }) => {
+                const dbp = await createConfiguredDbp(BROKER_PROTECTION_CONFIGS.default);
+                await dbp.navigatesTo(imageCaptchaTargetPage);
+                await dbp.receivesInlineAction(createSolveImageCaptchaAction({ selector: imageCaptchaResponseSelector }));
+                dbp.getSuccessResponse();
+
+                await dbp.isCaptchaTokenFilled(imageCaptchaResponseSelector);
+            });
+        });
+    });
 });

injected/integration-test/mocks/broker-protection/captcha.js

@@ -34,6 +34,18 @@ export function createGetRecaptchaInfoAction(actionOverrides = {}) {
     });
 }
 
+/**
+ * @param {Partial<PirAction>} [actionOverrides]
+ */
+export function createGetImageCaptchaInfoAction(actionOverrides = {}) {
+    return createGetCaptchaInfoAction({
+        action: {
+            captchaType: 'image',
+            ...actionOverrides,
+        },
+    });
+}
+
 /**
  * @param {object} params
  * @param {Omit<PirAction, 'id' | 'actionType'>} params.action
@@ -66,6 +78,18 @@ export function createSolveRecaptchaAction(actionOverrides = {}) {
     });
 }
 
+/**
+ * @param {Partial<PirAction>} [actionOverrides]
+ */
+export function createSolveImageCaptchaAction(actionOverrides = {}) {
+    return createSolveCaptchaAction({
+        action: {
+            captchaType: 'image',
+            ...actionOverrides,
+        },
+    });
+}
+
 // Captcha responses
 
 /**

injected/integration-test/page-objects/broker-protection.js

@@ -70,8 +70,8 @@ export class BrokerProtectionPage {
      * @return {Promise<void>}
      */
     async isCaptchaTokenFilled(responseElementSelector) {
-        const captchaTextArea = await this.page.$(responseElementSelector);
-        const captchaToken = await captchaTextArea?.evaluate((element) => element.innerHTML);
+        const captchaTarget = await this.page.$(responseElementSelector);
+        const captchaToken = await captchaTarget?.evaluate((element) => ('value' in element ? element.value : element.innerHTML));
         expect(captchaToken).toBe('test_token');
     }
 
@@ -99,7 +99,17 @@ export class BrokerProtectionPage {
      */
     isCaptchaMatch(response, { captchaType, targetPage }) {
         const expectedResponse = createCaptchaResponse({ captchaType, targetPage });
-        expect(response).toStrictEqual(expectedResponse);
+
+        switch (captchaType) {
+            case 'image':
+                // Validate that the correct keys are present in the response
+                expect(Object.keys(response).sort()).toStrictEqual(Object.keys(expectedResponse).sort());
+                // Validate that the siteKey looks like a base64 encoded image
+                expect(response.siteKey).toMatch(/^data:image\/jpeg;base64,/);
+                break;
+            default:
+                expect(response).toStrictEqual(expectedResponse);
+        }
     }
 
     async isCaptchaError() {

injected/integration-test/test-pages/broker-protection/pages/image-captcha.html

@@ -1,6 +1,6 @@
-export { extract } from './extract';
-export { fillForm } from './fill-form';
-export { click } from './click';
-export { expectation } from './expectation';
-export { navigate } from './navigate';
-export { getCaptchaInfo, solveCaptcha } from '../captcha-services/captcha.service';
+export { extract } from './extract.js';
+export { fillForm } from './fill-form.js';
+export { click } from './click.js';
+export { expectation } from './expectation.js';
+export { navigate } from './navigate.js';
+export { getCaptchaInfo, solveCaptcha } from '../captcha-services/captcha.service.js';

injected/src/features/broker-protection/actions/actions.js

@@ -51,9 +51,9 @@ export function getSupportingCodeToInject(action) {
  *
  * @param {import('../types.js').PirAction} action
  * @param {Document | HTMLElement} root
- * @return {import('../types.js').ActionResponse}
+ * @return {Promise<import('../types.js').ActionResponse>}
  */
-export function getCaptchaInfo(action, root = document) {
+export async function getCaptchaInfo(action, root = document) {
     const { id: actionID, actionType, captchaType, selector } = action;
     if (!captchaType) {
         // ensures backward compatibility with old actions
@@ -72,7 +72,7 @@ export function getCaptchaInfo(action, root = document) {
         return createError(captchaProvider.error.message);
     }
 
-    const captchaIdentifier = captchaProvider.getCaptchaIdentifier(captchaContainer);
+    const captchaIdentifier = await captchaProvider.getCaptchaIdentifier(captchaContainer);
     if (!captchaIdentifier) {
         return createError(`could not extract captcha identifier from the container with selector ${selector}`);
     }

injected/src/features/broker-protection/captcha-services/captcha.service.js

@@ -22,7 +22,7 @@ export class CloudFlareTurnstileProvider {
      */
     getCaptchaIdentifier(_captchaContainerElement) {
         // TODO: Implement
-        return null;
+        return Promise.resolve(null);
     }
 
     getSupportingCodeToInject() {

injected/src/features/broker-protection/captcha-services/providers/cloudflare-turnstile.js

@@ -22,7 +22,7 @@ export class HCaptchaProvider {
      */
     getCaptchaIdentifier(_captchaContainerElement) {
         // TODO: Implement
-        return null;
+        return Promise.resolve(null);
     }
 
     getSupportingCodeToInject() {

injected/src/features/broker-protection/captcha-services/providers/hcaptcha.js

@@ -0,0 +1,90 @@
+import { PirError } from '../../types';
+import { svgToBase64Jpg, imageToBase64 } from '../utils/image';
+import { injectTokenIntoElement } from '../utils/token';
+import { isElementType } from '../utils/element';
+import { stringifyFunction } from '../utils/stringify-function';
+
+/**
+ * @import { CaptchaProvider } from './provider.interface';
+ * @implements {CaptchaProvider}
+ */
+export class ImageProvider {
+    getType() {
+        return 'image';
+    }
+
+    /**
+     * @param {HTMLElement} captchaImageElement - The captcha image element
+     */
+    isSupportedForElement(captchaImageElement) {
+        if (!captchaImageElement) {
+            return false;
+        }
+
+        return isElementType(captchaImageElement, ['img', 'svg']);
+    }
+
+    /**
+     * @param {HTMLElement} captchaImageElement - The captcha image element
+     */
+    async getCaptchaIdentifier(captchaImageElement) {
+        if (isSVGElement(captchaImageElement)) {
+            return await svgToBase64Jpg(captchaImageElement);
+        }
+
+        if (isImgElement(captchaImageElement)) {
+            return imageToBase64(captchaImageElement);
+        }
+
+        return PirError.create(
+            `[ImageProvider.getCaptchaIdentifier] could not extract Base64 from image with tag name: ${captchaImageElement.tagName}`,
+        );
+    }
+
+    getSupportingCodeToInject() {
+        return null;
+    }
+
+    /**
+     * @param {HTMLElement} captchaInputElement - The captcha input element
+     */
+    canSolve(captchaInputElement) {
+        return isElementType(captchaInputElement, ['input', 'textarea']);
+    }
+
+    /**
+     * @param {HTMLInputElement} captchaInputElement - The captcha input element
+     * @param {string} token - The solved captcha token
+     */
+    injectToken(captchaInputElement, token) {
+        return injectTokenIntoElement({ captchaInputElement, token });
+    }
+
+    /**
+     * @param {HTMLElement} _captchaInputElement - The element containing the captcha
+     * @param {string} _token - The solved captcha token
+     */
+    getSolveCallback(_captchaInputElement, _token) {
+        return stringifyFunction({
+            functionBody: function callbackNoop() {},
+            functionName: 'callbackNoop',
+            args: {},
+        });
+    }
+}
+
+/**
+ * @param {HTMLElement} element
+ * @return {element is SVGElement}
+ */
+function isSVGElement(element) {
+    return isElementType(element, 'svg');
+}
+
+/**
+ * @param {HTMLElement} element
+ * @return {element is HTMLImageElement}
+ */
+function isImgElement(element) {
+    return isElementType(element, 'img');
+}

injected/src/features/broker-protection/captcha-services/providers/image.js

@@ -28,10 +28,10 @@ export class CaptchaProvider {
      * Extracts the site key from the captcha container element
      * @abstract
      * @param {HTMLElement} _captchaContainerElement - The element containing the captcha
-     * @returns {PirError | string | null} The site key or null if not found
+     * @returns {Promise<PirError | string | null>} The site key or null if not found
      */
     getCaptchaIdentifier(_captchaContainerElement) {
-        throw new Error('getCaptchaIdentifier() missing implementation');
+        return Promise.reject(new Error('getCaptchaIdentifier() missing implementation'));
     }
 
     /**

injected/src/features/broker-protection/captcha-services/providers/provider.interface.js

@@ -46,9 +46,11 @@ export class ReCaptchaProvider {
      * @param {HTMLElement} captchaContainerElement
      */
     getCaptchaIdentifier(captchaContainerElement) {
-        return safeCallWithError(
-            () => getSiteKeyFromSearchParam({ captchaElement: this._getCaptchaElement(captchaContainerElement), siteKeyAttrName: 'k' }),
-            { errorMessage: '[ReCaptchaProvider.getCaptchaIdentifier] could not extract site key' },
+        return Promise.resolve(
+            safeCallWithError(
+                () => getSiteKeyFromSearchParam({ captchaElement: this._getCaptchaElement(captchaContainerElement), siteKeyAttrName: 'k' }),
+                { errorMessage: '[ReCaptchaProvider.getCaptchaIdentifier] could not extract site key' },
+            ),
         );
     }
 

injected/src/features/broker-protection/captcha-services/providers/recaptcha.js

@@ -1,5 +1,6 @@
 import { CaptchaFactory } from '../factory';
 import { ReCaptchaProvider } from './recaptcha';
+import { ImageProvider } from './image';
 
 const captchaFactory = new CaptchaFactory();
 
@@ -19,4 +20,6 @@ captchaFactory.registerProvider(
     }),
 );
 
+captchaFactory.registerProvider(new ImageProvider());
+
 export { captchaFactory };

injected/src/features/broker-protection/captcha-services/providers/registry.js

@@ -0,0 +1,13 @@
+/**
+ *
+ * @param {HTMLElement} element - The element to check
+ * @param {string | string[]} tag - The tag name(s) to check against
+ * @returns {boolean} - True if the element is of the specified tag name(s), false otherwise
+ */
+export function isElementType(element, tag) {
+    if (Array.isArray(tag)) {
+        return tag.some((t) => isElementType(element, t));
+    }
+
+    return element.tagName.toLowerCase() === tag.toLowerCase();
+}

injected/src/features/broker-protection/captcha-services/utils/element.js

@@ -0,0 +1,65 @@
+/**
+ * Converts an SVG element to a base64-encoded JPEG string.
+ *
+ * @param {SVGElement} svgElement - The SVG element to convert
+ * @param {string} [backgroundColor='white'] - The background color for the JPEG image
+ * @return {Promise<string>} - A promise that resolves to the base64-encoded JPEG image
+ */
+export function svgToBase64Jpg(svgElement, backgroundColor = 'white') {
+    const svgString = new XMLSerializer().serializeToString(svgElement);
+    const svgDataUrl = 'data:image/svg+xml;base64,' + btoa(svgString);
+
+    return new Promise((resolve, reject) => {
+        const img = new Image();
+        img.onload = () => {
+            const canvas = document.createElement('canvas');
+            const ctx = canvas.getContext('2d');
+
+            if (!ctx) {
+                reject(new Error('Could not get 2D context from canvas'));
+                return;
+            }
+
+            canvas.width = img.width;
+            canvas.height = img.height;
+
+            ctx.fillStyle = backgroundColor;
+            ctx.fillRect(0, 0, canvas.width, canvas.height);
+            ctx.drawImage(img, 0, 0);
+
+            const jpgBase64 = canvas.toDataURL('image/jpeg');
+
+            resolve(jpgBase64);
+        };
+        img.onerror = (error) => {
+            reject(error);
+        };
+
+        img.src = svgDataUrl;
+    });
+}
+
+/**
+ * Converts an image element to a base64-encoded JPEG string
+ *
+ * @param {HTMLImageElement} imageElement - The image element to convert.
+ * @return {string} - The base64-encoded JPEG string.
+ * @throws {Error} - Throws an error if the canvas context cannot be obtained.
+ */
+export function imageToBase64(imageElement) {
+    const canvas = document.createElement('canvas');
+    const ctx = canvas.getContext('2d');
+
+    if (!ctx) {
+        throw Error('[imageToBase64] Could not get 2D context from canvas');
+    }
+
+    canvas.width = imageElement.width;
+    canvas.height = imageElement.height;
+
+    ctx.drawImage(imageElement, 0, 0, canvas.width, canvas.height);
+
+    const base64String = canvas.toDataURL('image/jpeg'); // You can change the format if needed
+
+    return base64String;
+}