ntp: batched api

Author: shakyShane

special-pages/pages/new-tab/app/activity/ActivityProvider.js

@@ -4,10 +4,11 @@ import { useMessaging } from '../types.js';
 import { ActivityService } from './activity.service.js';
 import { reducer, useConfigSubscription, useInitialDataAndConfig } from '../service.hooks.js';
 import { eventToTarget } from '../utils.js';
-import { usePlatformName } from '../settings.provider.js';
+import { useBatchedActivityApi, usePlatformName } from '../settings.provider.js';
 import { ACTION_ADD_FAVORITE, ACTION_BURN, ACTION_REMOVE, ACTION_REMOVE_FAVORITE } from './constants.js';
 import { batch, signal, useSignal, useSignalEffect } from '@preact/signals';
 import { DDG_DEFAULT_ICON_SIZE } from '../favorites/constants.js';
+import { BatchedActivity } from './batched-activity.service.js';
 
 /**
  * @typedef {import('../../types/new-tab.js').ActivityData} ActivityData
@@ -54,9 +55,10 @@ export function ActivityProvider(props) {
 
     const [state, dispatch] = useReducer(reducer, initial);
     const platformName = usePlatformName();
+    const batched = useBatchedActivityApi();
 
     // create an instance of `ActivityService` for the lifespan of this component.
-    const service = useService();
+    const service = useService(batched);
 
     // get initial data
     useInitialDataAndConfig({ dispatch, service });
@@ -142,64 +144,81 @@ export function ActivityProvider(props) {
  */
 function normalizeItems(prev, data) {
     return {
-        favorites: Object.fromEntries(
-            data.activity.map((x) => {
-                return [x.url, x.favorite];
-            }),
-        ),
-        items: Object.fromEntries(
-            data.activity.map((x) => {
-                /** @type {Item} */
-                const next = {
-                    etldPlusOne: x.etldPlusOne,
-                    title: x.title,
-                    url: x.url,
-                    faviconMax: x.favicon?.maxAvailableSize ?? DDG_DEFAULT_ICON_SIZE,
-                    favoriteSrc: x.favicon?.src,
-                    trackersFound: x.trackersFound,
-                };
-                const differs = shallowDiffers(next, prev.items[x.url] || {});
-                return [x.url, differs ? next : prev.items[x.url] || {}];
-            }),
-        ),
-        history: Object.fromEntries(
-            data.activity.map((x) => {
-                const differs = shallowDiffers(x.history, prev.history[x.url] || []);
-                return [x.url, differs ? [...x.history] : prev.history[x.url] || []];
-            }),
-        ),
-        trackingStatus: Object.fromEntries(
-            data.activity.map((x) => {
-                const prevItem = prev.trackingStatus[x.url] || {
-                    totalCount: 0,
-                    trackerCompanies: [],
-                };
-                const differs = shallowDiffers(x.trackingStatus.trackerCompanies, prevItem.trackerCompanies);
-                if (prevItem.totalCount !== x.trackingStatus.totalCount || differs) {
+        favorites: {
+            ...prev.favorites,
+            ...Object.fromEntries(
+                data.activity.map((x) => {
+                    return [x.url, x.favorite];
+                }),
+            ),
+        },
+        items: {
+            ...prev.items,
+            ...Object.fromEntries(
+                data.activity.map((x) => {
+                    /** @type {Item} */
                     const next = {
-                        totalCount: x.trackingStatus.totalCount,
-                        trackerCompanies: [...x.trackingStatus.trackerCompanies],
+                        etldPlusOne: x.etldPlusOne,
+                        title: x.title,
+                        url: x.url,
+                        faviconMax: x.favicon?.maxAvailableSize ?? DDG_DEFAULT_ICON_SIZE,
+                        favoriteSrc: x.favicon?.src,
+                        trackersFound: x.trackersFound,
                     };
-                    return [x.url, next];
-                }
-                return [x.url, prevItem];
-            }),
-        ),
+                    const differs = shallowDiffers(next, prev.items[x.url] || {});
+                    return [x.url, differs ? next : prev.items[x.url] || {}];
+                }),
+            ),
+        },
+        history: {
+            ...prev.history,
+            ...Object.fromEntries(
+                data.activity.map((x) => {
+                    const differs = shallowDiffers(x.history, prev.history[x.url] || []);
+                    return [x.url, differs ? [...x.history] : prev.history[x.url] || []];
+                }),
+            ),
+        },
+        trackingStatus: {
+            ...prev.trackingStatus,
+            ...Object.fromEntries(
+                data.activity.map((x) => {
+                    const prevItem = prev.trackingStatus[x.url] || {
+                        totalCount: 0,
+                        trackerCompanies: [],
+                    };
+                    const differs = shallowDiffers(x.trackingStatus.trackerCompanies, prevItem.trackerCompanies);
+                    if (prevItem.totalCount !== x.trackingStatus.totalCount || differs) {
+                        const next = {
+                            totalCount: x.trackingStatus.totalCount,
+                            trackerCompanies: [...x.trackingStatus.trackerCompanies],
+                        };
+                        return [x.url, next];
+                    }
+                    return [x.url, prevItem];
+                }),
+            ),
+        },
     };
 }
 
 /**
- * @param {{ available: string[]; max: number }} prev
- * @param {ActivityData} data
- * @return {{ available: string[]; max: number }}
+ * @param {string[]} prev
+ * @param {string[]} data
+ * @return {string[]}
  */
 function normalizeKeys(prev, data) {
-    const keys = data.activity.map((x) => x.url);
-    const next = shallowDiffers(prev, keys) ? keys : prev.available;
-    return {
-        available: next,
-        max: keys.length,
-    };
+    const next = shallowDiffers(prev, data) ? [...data] : prev;
+    return next;
+}
+
+/**
+ * @param {string[]} prev
+ * @param {import('../../types/new-tab.js').UrlInfo} data
+ * @return {string[]}
+ */
+function normalizeUrls(prev, data) {
+    return shallowDiffers(prev, data.urls) ? [...data.urls] : prev;
 }
 
 /**
@@ -216,16 +235,23 @@ export function shallowDiffers(a, b) {
 
 export const SignalStateContext = createContext({
     activity: signal(/** @type {NormalizedActivity} */ ({})),
-    keys: signal(/** @type {{available: string[]; max: number}} */ ({ available: [], max: 0 })),
+    keys: signal(/** @type {string[]} */ ([])),
 });
 
 export function SignalStateProvider({ children }) {
     const { state } = useContext(ActivityContext);
-    const service = useContext(ActivityServiceContext);
+    const service = /** @type {BatchedActivity} */ (useContext(ActivityServiceContext));
     if (state.status !== 'ready') throw new Error('must have ready status here');
     if (!service) throw new Error('must have service here');
+    if (!service.urlService.data) throw new Error('must have initialised the url service by this point');
 
-    const keys = useSignal(normalizeKeys({ available: [], max: 0 }, state.data));
+    const keys = useSignal(
+        normalizeKeys(
+            [],
+            state.data.activity.map((x) => x.url),
+        ),
+    );
+    const urls = useSignal(normalizeUrls([], service.urlService.data));
     const activity = useSignal(
         normalizeItems(
             {
@@ -240,23 +266,69 @@ export function SignalStateProvider({ children }) {
 
     useSignalEffect(() => {
         if (!service) return console.warn('could not access service');
-        const unsub = service.onData((evt) => {
+        const src = /** @type {BatchedActivity} */ (service);
+        const unsub = src.onData((evt) => {
             batch(() => {
-                keys.value = normalizeKeys(keys.value, evt.data);
                 activity.value = normalizeItems(activity.value, evt.data);
+                // const data = Object.keys(activity.value.items);
             });
         });
-        const handler = () => {
-            if (document.visibilityState === 'visible') {
-                console.log('will fetch');
-                service
-                    .triggerDataFetch()
-                    // eslint-disable-next-line promise/prefer-await-to-then
-                    .catch((e) => console.error('trigger fetch errored', e));
+        const unsubPatch = src.onUrlData((evt) => {
+            if (evt.data.patch) {
+                activity.value = normalizeItems(activity.value, { activity: [evt.data.patch] });
             }
+            urls.value = normalizeUrls(urls.value, evt.data);
+            const visible = keys.value;
+            const all = urls.value;
+            const nextVisibleRange = all.slice(0, visible.length);
+            setVisibleRange(nextVisibleRange);
+        });
+
+        /**
+         * @param {string[]} nextVisibleRange
+         */
+        function setVisibleRange(nextVisibleRange) {
+            keys.value = normalizeKeys(keys.value, nextVisibleRange);
+            fillHoles();
+            // todo, evict entries?
+        }
+
+        function fillHoles() {
+            const visible = keys.value;
+            const data = Object.keys(activity.value.items);
+            const missing = visible.filter((x) => !data.includes(x));
+            src.next(missing);
+        }
+
+        function updateVisible() {
+            const visibleLength = keys.value.length;
+            const end = visibleLength + service.SIZE;
+            const nextVisibleRange = urls.value.slice(0, end);
+            setVisibleRange(nextVisibleRange);
+        }
+
+        window.addEventListener('activity.next', updateVisible);
+
+        return () => {
+            unsub();
+            unsubPatch();
+            window.removeEventListener('activity.next', updateVisible);
+        };
+    });
+
+    useEffect(() => {
+        const handler = () => {
+            // todo: re-enable this
+            // if (document.visibilityState === 'visible') {
+            //     // console.log('will fetch');
+            //     src.triggerDataFetch()
+            //         // eslint-disable-next-line promise/prefer-await-to-then
+            //         .catch((e) => console.error('trigger fetch errored', e));
+            // }
         };
 
-        (() => {
+        // eslint-disable-next-line no-labels,no-unused-labels
+        $INTEGRATION: (() => {
             // export the event in tests
             if (window.__playwright_01) {
                 /** @type {any} */ (window).__trigger_document_visibilty__ = handler;
@@ -265,26 +337,26 @@ export function SignalStateProvider({ children }) {
 
         document.addEventListener('visibilitychange', handler);
         return () => {
-            unsub();
             document.removeEventListener('visibilitychange', handler);
         };
-    });
+    }, []);
 
     return <SignalStateContext.Provider value={{ activity, keys }}>{children}</SignalStateContext.Provider>;
 }
 
 /**
- * @return {import("preact").RefObject<ActivityService>}
+ * @param {boolean} useBatched
+ * @return {import("preact").RefObject<ActivityService|BatchedActivity>}
  */
-export function useService() {
-    const service = useRef(/** @type {ActivityService|null} */ (null));
+export function useService(useBatched) {
+    const service = useRef(/** @type {ActivityService|BatchedActivity|null} */ (null));
     const ntp = useMessaging();
     useEffect(() => {
-        const stats = new ActivityService(ntp);
+        const stats = useBatched ? new BatchedActivity(ntp) : new ActivityService(ntp);
         service.current = stats;
         return () => {
             stats.destroy();
         };
-    }, [ntp]);
+    }, [ntp, useBatched]);
     return service;
 }

special-pages/pages/new-tab/app/activity/activity.md

@@ -62,7 +62,35 @@ Notes:
 - {@link "NewTab Messages".ActivityGetConfigRequest}
 - Used to fetch the initial config data (eg: expanded vs collapsed)
 - returns {@link "NewTab Messages".ActivityConfig}
-- 
+
+### `activity_getUrls`
+- {@link "NewTab Messages".ActivityGetUrlsRequest}
+- Used to fetch the initial config data (eg: expanded vs collapsed)
+- returns {@link "NewTab Messages".UrlInfo}
+
+```json
+{
+  "urls": ["..."],
+  "totalTrackersBlocked": 123
+}
+```
+
+### `activity_getDataForUrls`
+- {@link "NewTab Messages".ActivityGetDataForUrlsRequest}
+- Used to confirm the burn action - native side may or may not show a modal
+- sends {@link "NewTab Messages".DataForUrlsParams}
+- returns {@link "NewTab Messages".ActivityData}
+- Note: This response is the same format as `activity_getData`, where DomainActivity items are delivered under `.activity`
+
+```json
+{
+  "activity": [
+    {"...":  "..."}
+  ]
+}
+```
+
+
 ### `activity_confirmBurn`
 - {@link "NewTab Messages".ActivityConfirmBurnRequest}
 - Used to confirm the burn action - native side may or may not show a modal
@@ -93,6 +121,27 @@ by sending the notification `activity_burnAnimationComplete`
 - The activity data used in the feed.
 - returns {@link "NewTab Messages".ActivityData}
 
+### `activity_onDataPatch`
+- {@link "NewTab Messages".ActivityOnDataPatchSubscription}
+- The activity data used in the feed.
+- returns {@link "NewTab Messages".UrlInfo} + optional {@link "NewTab Messages".PatchData}
+
+```json
+{
+  "urls": ["..."],
+  "totalTrackersBlocked": 123
+}
+```
+```json
+{
+  "urls": ["..."],
+  "totalTrackersBlocked": 123,
+  "patch": {
+    "...": "..."
+  }
+}
+```
+
 ### `activity_onConfigUpdate` 
 - {@link "NewTab Messages".ActivityOnDataUpdateSubscription }
 - The widget config

special-pages/pages/new-tab/app/activity/activity.service.js

@@ -16,7 +16,6 @@ export class ActivityService {
             initial: () => ntp.messaging.request('activity_getData'),
             subscribe: (cb) => ntp.messaging.subscribe('activity_onDataUpdate', cb),
         });
-
         /** @type {Service<ActivityConfig>} */
         this.configService = new Service({
             initial: () => ntp.messaging.request('activity_getConfig'),