Storage List API (#1610)

Author: fredzqm

packages/firebase/index.d.ts

@@ -5484,6 +5484,85 @@ declare namespace firebase.storage {
      *     including if the object did not exist.
      */
     updateMetadata(metadata: firebase.storage.SettableMetadata): Promise<any>;
+    /**
+     * List all items (files) and prefixes (folders) under this storage reference.
+     *
+     * This is a helper method for calling list() repeatedly until there are
+     * no more results. The default pagination size is 1000.
+     *
+     * Note: The results may not be consistent if objects are changed while this
+     * operation is running.
+     *
+     * Warning: listAll may potentially consume too many resources if there are
+     * too many results.
+     *
+     * @return A Promise that resolves with all the items and prefixes under
+     *      the current storage reference. `prefixes` contains references to
+     *      sub-directories and `items` contains references to objects in this
+     *      folder. `nextPageToken` is never returned.
+     */
+    listAll(): Promise<ListResult>;
+    /**
+     * List items (files) and prefixes (folders) under this storage reference.
+     *
+     * List API is only available for Firebase Rules Version 2.
+     *
+     * GCS is a key-blob store. Firebase Storage imposes the semantic of '/'
+     * delimited folder structure.
+     * Refer to GCS's List API if you want to learn more.
+     *
+     * To adhere to Firebase Rules's Semantics, Firebase Storage does not
+     * support objects whose paths end with "/" or contain two consecutive
+     * "/"s. Firebase Storage List API will filter these unsupported objects.
+     * list() may fail if there are too many unsupported objects in the bucket.
+     *
+     * @param options See ListOptions for details.
+     * @return A Promise that resolves with the items and prefixes.
+     *      `prefixes` contains references to sub-folders and `items`
+     *      contains references to objects in this folder. `nextPageToken`
+     *      can be used to get the rest of the results.
+     */
+    list(options?: ListOptions): Promise<ListResult>;
+  }
+
+  /**
+   * Result returned by list().
+   */
+  interface ListResult {
+    /**
+     * References to prefixes (sub-folders). You can call list() on them to
+     * get its contents.
+     *
+     * Folders are implicit based on '/' in the object paths.
+     * For example, if a bucket has two objects '/a/b/1' and '/a/b/2', list('/a')
+     * will return '/a/b' as a prefix.
+     */
+    prefixes: Reference[];
+    /**
+     * Objects in this directory.
+     * You can call getMetadate() and getDownloadUrl() on them.
+     */
+    items: Reference[];
+    /**
+     * If set, there might be more results for this list. Use this token to resume the list.
+     */
+    nextPageToken: string | null;
+  }
+
+  /**
+   * The options list() accepts.
+   */
+  interface ListOptions {
+    /**
+     * If set, limits the total number of `prefixes` and `items` to return.
+     * The default and maximum maxResults is 1000.
+     */
+    maxResults?: number | null;
+    /**
+     * The `nextPageToken` from a previous call to list(). If provided,
+     * listing is resumed from the previous position.
+     */
+    pageToken?: string | null;
   }
 
   /**

packages/storage-types/index.d.ts

@@ -51,6 +51,18 @@ export interface Reference {
   storage: Storage;
   toString(): string;
   updateMetadata(metadata: SettableMetadata): Promise<FullMetadata>;
+  list(options?: ListOptions): Promise<ListResult>;
+}
+
+export interface ListResult {
+  prefixes: Reference[];
+  items: Reference[];
+  nextPageToken: string | null;
+}
+
+export interface ListOptions {
+  maxResults?: number | null;
+  pageToken?: string | null;
 }
 
 export interface SettableMetadata {

packages/storage/CHANGELOG.md

@@ -0,0 +1,2 @@
+- [Feature] Added the support for List API.
+

packages/storage/src/implementation/args.ts

@@ -15,8 +15,8 @@
  * limitations under the License.
  */
 import * as errorsExports from './error';
-import { errors } from './error';
 import * as MetadataUtils from './metadata';
+import * as ListOptionsUtils from './list';
 import * as type from './type';
 
 /**
@@ -117,6 +117,10 @@ export function metadataSpec(opt_optional?: boolean): ArgSpec {
   return new ArgSpec(MetadataUtils.metadataValidator, opt_optional);
 }
 
+export function listOptionSpec(opt_optional?: boolean): ArgSpec {
+  return new ArgSpec(ListOptionsUtils.listOptionsValidator, opt_optional);
+}
+
 export function nonNegativeNumberSpec(): ArgSpec {
   function validator(p: any) {
     let valid = type.isNumber(p) && p >= 0;

packages/storage/src/implementation/list.ts

@@ -0,0 +1,115 @@
+/**
+ * @license
+ * Copyright 2019 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @fileoverview Documentation for the listOptions and listResult format
+ */
+import { AuthWrapper } from './authwrapper';
+import { Location } from './location';
+import * as json from './json';
+import * as type from './type';
+import { ListResult } from '../list';
+
+/**
+ * Represents the simplified object metadata returned by List API.
+ * Other fields are filtered because list in Firebase Rules does not grant
+ * the permission to read the metadata.
+ */
+interface ListMetadataResponse {
+  name: string;
+  bucket: string;
+}
+
+/**
+ * Represents the JSON response of List API.
+ */
+interface ListResultResponse {
+  prefixes: string[];
+  items: ListMetadataResponse[];
+  nextPageToken?: string;
+}
+
+const MAX_RESULTS_KEY = 'maxResults';
+const MAX_MAX_RESULTS = 1000;
+const PAGE_TOKEN_KEY = 'pageToken';
+const PREFIXES_KEY = 'prefixes';
+const ITEMS_KEY = 'items';
+
+function fromBackendResponse(
+  authWrapper: AuthWrapper,
+  resource: ListResultResponse
+): ListResult {
+  const listResult: ListResult = {
+    prefixes: [],
+    items: [],
+    nextPageToken: resource['nextPageToken']
+  };
+  if (resource[PREFIXES_KEY]) {
+    for (const path of resource[PREFIXES_KEY]) {
+      const pathWithoutTrailingSlash = path.replace(/\/$/, '');
+      const reference = authWrapper.makeStorageReference(
+        new Location(authWrapper.bucket(), pathWithoutTrailingSlash)
+      );
+      listResult.prefixes.push(reference);
+    }
+  }
+
+  if (resource[ITEMS_KEY]) {
+    for (const item of resource[ITEMS_KEY]) {
+      const reference = authWrapper.makeStorageReference(
+        new Location(authWrapper.bucket(), item['name'])
+      );
+      listResult.items.push(reference);
+    }
+  }
+  return listResult;
+}
+
+export function fromResponseString(
+  authWrapper: AuthWrapper,
+  resourceString: string
+): ListResult | null {
+  const obj = json.jsonObjectOrNull(resourceString);
+  if (obj === null) {
+    return null;
+  }
+  const resource = obj as ListResultResponse;
+  return fromBackendResponse(authWrapper, resource);
+}
+
+export function listOptionsValidator(p: any) {
+  const validType = p && type.isObject(p);
+  if (!validType) {
+    throw 'Expected ListOptions object.';
+  }
+  for (const key in p) {
+    if (key === MAX_RESULTS_KEY) {
+      if (!type.isInteger(p[MAX_RESULTS_KEY]) || p[MAX_RESULTS_KEY] <= 0) {
+        throw 'Expected maxResults to be a positive number.';
+      }
+      if (p[MAX_RESULTS_KEY] > 1000) {
+        throw `Expected maxResults to be less than or equal to ${MAX_MAX_RESULTS}.`;
+      }
+    } else if (key === PAGE_TOKEN_KEY) {
+      if (p[PAGE_TOKEN_KEY] && !type.isString(p[PAGE_TOKEN_KEY])) {
+        throw 'Expected pageToken to be string.';
+      }
+    } else {
+      throw 'Unknown option: ' + key;
+    }
+  }
+}

packages/storage/src/implementation/location.ts

@@ -36,6 +36,10 @@ export class Location {
     return this.path_;
   }
 
+  get isRoot(): boolean {
+    return this.path.length === 0;
+  }
+
   fullServerUrl(): string {
     let encode = encodeURIComponent;
     return '/b/' + encode(this.bucket) + '/o/' + encode(this.path);

packages/storage/src/implementation/requests.ts

@@ -20,6 +20,7 @@
  */
 
 import { Metadata } from '../metadata';
+import { ListResult } from '../list';
 
 import * as array from './array';
 import { AuthWrapper } from './authwrapper';
@@ -28,6 +29,7 @@ import * as errorsExports from './error';
 import { FirebaseStorageError } from './error';
 import { Location } from './location';
 import * as MetadataUtils from './metadata';
+import * as ListResultUtils from './list';
 import * as object from './object';
 import { RequestInfo } from './requestinfo';
 import * as type from './type';
@@ -59,6 +61,17 @@ export function metadataHandler(
   return handler;
 }
 
+export function listHandler(
+  authWrapper: AuthWrapper
+): (p1: XhrIo, p2: string) => ListResult {
+  function handler(xhr: XhrIo, text: string): ListResult {
+    const listResult = ListResultUtils.fromResponseString(authWrapper, text);
+    handlerCheck(listResult !== null);
+    return listResult as ListResult;
+  }
+  return handler;
+}
+
 export function downloadUrlHandler(
   authWrapper: AuthWrapper,
   mappings: MetadataUtils.Mappings
@@ -143,6 +156,43 @@ export function getMetadata(
   return requestInfo;
 }
 
+export function list(
+  authWrapper: AuthWrapper,
+  location: Location,
+  delimiter?: string,
+  pageToken?: string,
+  maxResults?: number
+): RequestInfo<ListResult> {
+  let urlParams = {};
+  if (location.isRoot) {
+    urlParams['prefix'] = '';
+  } else {
+    urlParams['prefix'] = location.path + '/';
+  }
+  if (delimiter && delimiter.length > 0) {
+    urlParams['delimiter'] = delimiter;
+  }
+  if (pageToken) {
+    urlParams['pageToken'] = pageToken;
+  }
+  if (maxResults) {
+    urlParams['maxResults'] = maxResults;
+  }
+  const urlPart = location.bucketOnlyServerUrl();
+  const url = UrlUtils.makeUrl(urlPart);
+  const method = 'GET';
+  const timeout = authWrapper.maxOperationRetryTime();
+  const requestInfo = new RequestInfo(
+    url,
+    method,
+    listHandler(authWrapper),
+    timeout
+  );
+  requestInfo.urlParams = urlParams;
+  requestInfo.errorHandler = sharedErrorHandler(location);
+  return requestInfo;
+}
+
 export function getDownloadUrl(
   authWrapper: AuthWrapper,
   location: Location,

packages/storage/src/implementation/type.ts

@@ -46,6 +46,10 @@ export function isString(p: any): boolean {
   return typeof p === 'string' || p instanceof String;
 }
 
+export function isInteger(p: any): boolean {
+  return isNumber(p) && Number.isInteger(p);
+}
+
 export function isNumber(p: any): boolean {
   return typeof p === 'number' || p instanceof Number;
 }

packages/storage/src/list.ts

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2019 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Reference } from './reference';
+
+/**
+ * @fileoverview Documentation for ListOptions and ListResult format.
+ */
+export type ListOptions = {
+  maxResults?: number | null;
+  pageToken?: string | null;
+};
+
+export type ListResult = {
+  prefixes: Reference[];
+  items: Reference[];
+  nextPageToken?: string | null;
+};