Merging separate Dart and Flutter SDK indexes into a single index. (#8776)

Author: isoos

app/bin/tools/sdk_search_benchmark.dart

@@ -2,11 +2,11 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:pub_dev/search/flutter_sdk_mem_index.dart';
+import 'package:pub_dev/search/sdk_mem_index.dart';
 
 /// Loads a Dart SDK search snapshot and executes queries on it, benchmarking their total time to complete.
 Future<void> main() async {
-  final index = await createFlutterSdkMemIndex();
+  final index = await createSdkMemIndex();
 
   // NOTE: please add more queries to this list, especially if there is a performance bottleneck.
   final queries = [

app/lib/search/backend.dart

@@ -42,10 +42,9 @@ import '../task/backend.dart';
 import '../task/global_lock.dart';
 import '../task/models.dart';
 
-import 'dart_sdk_mem_index.dart';
-import 'flutter_sdk_mem_index.dart';
 import 'models.dart';
 import 'result_combiner.dart';
+import 'sdk_mem_index.dart';
 import 'search_client.dart';
 import 'search_service.dart';
 import 'text_utils.dart';
@@ -623,8 +622,7 @@ class _CombinedSearchIndex implements SearchIndex {
   PackageSearchResult search(ServiceSearchQuery query) {
     final combiner = SearchResultCombiner(
       primaryIndex: _packageIndexHolder._index,
-      dartSdkMemIndex: dartSdkMemIndex,
-      flutterSdkMemIndex: flutterSdkMemIndex,
+      sdkMemIndex: sdkMemIndex,
     );
     return combiner.search(query);
   }

app/lib/search/dart_sdk_mem_index.dart

@@ -13,13 +13,11 @@ import 'search_service.dart';
 /// SDK index.
 class SearchResultCombiner {
   final InMemoryPackageIndex primaryIndex;
-  final SdkMemIndex? dartSdkMemIndex;
-  final SdkMemIndex? flutterSdkMemIndex;
+  final SdkMemIndex? sdkMemIndex;
 
   SearchResultCombiner({
     required this.primaryIndex,
-    required this.dartSdkMemIndex,
-    required this.flutterSdkMemIndex,
+    required this.sdkMemIndex,
   });
 
   PackageSearchResult search(ServiceSearchQuery query) {
@@ -30,11 +28,10 @@ class SearchResultCombiner {
 
     final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') ||
         query.tagsPredicate.hasTag(SdkTag.sdkFlutter);
-    final sdkLibraryHits = [
-      ...?dartSdkMemIndex?.search(query.query!, limit: 2),
-      if (queryFlutterSdk)
-        ...?flutterSdkMemIndex?.search(query.query!, limit: 2),
-    ];
+    final sdkLibraryHits = sdkMemIndex
+            ?.search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk)
+            .toList() ??
+        <SdkLibraryHit>[];
     if (sdkLibraryHits.isNotEmpty) {
       // Do not display low SDK scores if the package hits are more relevant on the page.
       //

app/lib/search/flutter_sdk_mem_index.dart

@@ -4,17 +4,29 @@
 
 import 'dart:math';
 
+import 'package:gcloud/service_scope.dart' as ss;
+import 'package:logging/logging.dart';
 import 'package:meta/meta.dart';
 // ignore: implementation_imports
 import 'package:pana/src/dartdoc/dartdoc_index.dart';
 import 'package:path/path.dart' as p;
-import 'package:pub_dev/search/flutter_sdk_mem_index.dart';
+import 'package:pub_dev/search/backend.dart';
 
 import 'search_service.dart';
 import 'token_index.dart';
 
 export 'package:pana/src/dartdoc/dartdoc_index.dart';
 
+/// Sets the SDK in-memory index.
+void registerSdkMemIndex(SdkMemIndex? index) {
+  if (index != null) {
+    ss.register(#_sdkMemIndex, index);
+  }
+}
+
+/// The active SDK in-memory index.
+SdkMemIndex? get sdkMemIndex => ss.lookup(#_sdkMemIndex) as SdkMemIndex?;
+
 /// Results from these libraries are ranked with lower score and
 /// will be displayed only if the query has the library name, or
 /// there are not other results that could match the query.
@@ -30,48 +42,54 @@ const _defaultApiPageDirWeights = {
   'material/Icons': 0.25,
 };
 
+final _logger = Logger('search.dart_sdk_mem_index');
+final _dartUri = Uri.parse('https://api.dart.dev/stable/latest/');
+final _flutterUri = Uri.parse('https://api.flutter.dev/flutter/');
+
+/// Tries to load Dart and Flutter SDK's dartdoc `index.json` and build
+/// a search index from it.
+///
+/// Returns `null` when the loading of `index.json` failed, or when there
+/// was an error parsing the file or building the index.
+Future<SdkMemIndex?> createSdkMemIndex() async {
+  try {
+    final dartSdkContent =
+        await loadOrFetchSdkIndexJsonAsString(SdkMemIndex.dartSdkIndexJsonUri);
+    final flutterSdkContent = await loadOrFetchSdkIndexJsonAsString(
+        SdkMemIndex._flutterSdkIndexJsonUri);
+    return SdkMemIndex(
+      dartIndex: DartdocIndex.parseJsonText(dartSdkContent),
+      flutterIndex: DartdocIndex.parseJsonText(flutterSdkContent),
+    );
+  } catch (e, st) {
+    _logger.warning('Unable to load SDK index.', e, st);
+    return null;
+  }
+}
+
 /// In-memory index for SDK library search queries.
 class SdkMemIndex {
   final _libraries = <String, _Library>{};
   final Map<String, double> _apiPageDirWeights;
 
   SdkMemIndex({
-    required String sdk,
-    required Uri baseUri,
-    required DartdocIndex index,
-    Set<String>? allowedLibraries,
+    required DartdocIndex dartIndex,
+    required DartdocIndex flutterIndex,
     Map<String, double>? apiPageDirWeights,
   }) : _apiPageDirWeights = apiPageDirWeights ?? _defaultApiPageDirWeights {
-    _addDartdocIndex(sdk, baseUri, index, allowedLibraries);
-  }
-
-  static SdkMemIndex dart({required DartdocIndex index}) {
-    return SdkMemIndex(
-      sdk: 'dart',
-      baseUri: Uri.parse('https://api.dart.dev/stable/latest/'),
-      index: index,
-    );
-  }
-
-  factory SdkMemIndex.flutter({required DartdocIndex index}) {
-    return SdkMemIndex(
-      sdk: 'flutter',
-      baseUri: Uri.parse('https://api.flutter.dev/flutter/'),
-      index: index,
-      allowedLibraries: flutterSdkAllowedLibraries,
-    );
+    _addDartdocIndex('dart', _dartUri, dartIndex);
+    _addDartdocIndex('flutter', _flutterUri, flutterIndex);
   }
 
   static final dartSdkIndexJsonUri =
       Uri.parse('https://api.dart.dev/stable/latest/index.json');
-  static final flutterSdkIndexJsonUri =
+  static final _flutterSdkIndexJsonUri =
       Uri.parse('https://api.flutter.dev/flutter/index.json');
 
   void _addDartdocIndex(
     String sdk,
     Uri baseUri,
     DartdocIndex index,
-    Set<String>? allowedLibraries,
   ) {
     final textsPerLibrary = <String, Map<String, String>>{};
     final baseUris = <String, Uri>{};
@@ -81,11 +99,7 @@ class SdkMemIndex {
       final library = f.qualifiedName?.split('.').first;
       if (library == null) continue;
       if (f.href == null) continue;
-      if (allowedLibraries != null &&
-          allowedLibraries.isNotEmpty &&
-          !allowedLibraries.contains(library)) {
-        continue;
-      }
+      if (_libraries.containsKey(library)) continue;
       if (f.isLibrary) {
         baseUris[library] = baseUri.resolve(f.href!);
 
@@ -116,13 +130,15 @@ class SdkMemIndex {
   List<SdkLibraryHit> search(
     String query, {
     int? limit,
+    bool skipFlutter = false,
   }) {
     limit ??= 2;
     final words = query.split(' ').where((e) => e.isNotEmpty).toList();
     if (words.isEmpty) return <SdkLibraryHit>[];
 
     final hits = <_Hit>[];
     for (final library in _libraries.values) {
+      if (skipFlutter && library.isFlutter) continue;
       // We may reduce the rank of certain libraries, except when their name is
       // also part of the query. E.g. `dart:html` with `query=cursor` may be
       // scored lower than `query=html cursor`.
@@ -204,6 +220,7 @@ class _Library {
     required this.tokenIndex,
   });
 
+  late final isFlutter = sdk == 'flutter';
   late final weight = _libraryWeights[name] ?? 1.0;
   late final lastNamePart = name.split(':').last;
 }

app/lib/search/result_combiner.dart

@@ -9,8 +9,7 @@ import 'dart:isolate';
 import 'package:gcloud/service_scope.dart';
 import 'package:logging/logging.dart';
 import 'package:pub_dev/search/backend.dart';
-import 'package:pub_dev/search/dart_sdk_mem_index.dart';
-import 'package:pub_dev/search/flutter_sdk_mem_index.dart';
+import 'package:pub_dev/search/sdk_mem_index.dart';
 import 'package:pub_dev/search/search_service.dart';
 import 'package:pub_dev/search/updater.dart';
 import 'package:pub_dev/service/entrypoint/_isolate.dart';
@@ -37,8 +36,7 @@ Future<void> main(List<String> args, var message) async {
   }
   await fork(() async {
     await servicesWrapperFn(() async {
-      registerDartSdkMemIndex(await createDartSdkMemIndex());
-      registerFlutterSdkMemIndex(await createFlutterSdkMemIndex());
+      registerSdkMemIndex(await createSdkMemIndex());
       await indexUpdater.init();
 
       final requestReceivePort = ReceivePort();

app/lib/search/sdk_mem_index.dart

@@ -15,8 +15,10 @@ void main() {
     testWithProfile('fetch SDK library description', fn: () async {
       final content = await loadOrFetchSdkIndexJsonAsString(
           SdkMemIndex.dartSdkIndexJsonUri);
-      final index =
-          SdkMemIndex.dart(index: DartdocIndex.parseJsonText(content));
+      final index = SdkMemIndex(
+        dartIndex: DartdocIndex.parseJsonText(content),
+        flutterIndex: DartdocIndex([]),
+      );
       expect(
         index.getLibraryDescription('dart:async'),
         'Support for asynchronous programming, with classes such as Future and Stream.',

app/lib/service/entrypoint/search_index.dart

@@ -66,7 +66,10 @@ void main() {
       expect(parserWithoutFirstEntry, originalWithoutFirstEntry);
 
       // parsing into SDK index
-      final sdkMemIndex = SdkMemIndex.flutter(index: index);
+      final sdkMemIndex = SdkMemIndex(
+        dartIndex: DartdocIndex([]),
+        flutterIndex: index,
+      );
       final rs = sdkMemIndex.search('StatelessWidget');
       expect(json.decode(json.encode(rs)), [
         {

app/test/search/backend_test.dart

@@ -9,7 +9,6 @@ import 'package:pub_dev/search/mem_index.dart';
 import 'package:pub_dev/search/result_combiner.dart';
 import 'package:pub_dev/search/sdk_mem_index.dart';
 import 'package:pub_dev/search/search_service.dart';
-import 'package:pub_dev/shared/versions.dart';
 import 'package:test/test.dart';
 
 void main() {
@@ -30,10 +29,8 @@ void main() {
     );
     final combiner = SearchResultCombiner(
       primaryIndex: primaryIndex,
-      dartSdkMemIndex: SdkMemIndex(
-        sdk: 'dart',
-        baseUri: Uri.parse('https://api.dart.dev/stable/$runtimeSdkVersion/'),
-        index: DartdocIndex.fromJsonList([
+      sdkMemIndex: SdkMemIndex(
+        dartIndex: DartdocIndex.fromJsonList([
           {
             'name': 'dart:core',
             'qualifiedName': 'dart:core',
@@ -71,8 +68,8 @@ void main() {
             'enclosedBy': {'name': 'String', 'kind': 3}
           },
         ]),
+        flutterIndex: DartdocIndex([]),
       ),
-      flutterSdkMemIndex: null,
     );
 
     test('non-text ranking', () async {