Skip to content

优化缓存调用逻辑 #512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 28, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "qiniu-js",
"jsName": "qiniu",
"version": "3.2.0",
"version": "3.3.0",
"private": false,
"description": "Javascript SDK for Qiniu Resource (Cloud) Storage AP",
"main": "lib/index.js",
Expand Down Expand Up @@ -43,7 +43,7 @@
"@babel/plugin-proposal-object-rest-spread": "^7.10.1",
"@babel/plugin-transform-runtime": "^7.10.1",
"@babel/preset-env": "^7.10.2",
"@qiniu/eslint-config": "0.0.6-beta.7",
"@qiniu/eslint-config": "^0.0.6-beta.7",
"@types/jest": "^26.0.23",
"@types/node": "^15.3.1",
"@types/spark-md5": "^3.0.2",
Expand Down
4 changes: 2 additions & 2 deletions src/upload/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -330,8 +330,8 @@ export default abstract class Base {
return {
size,
loaded,
fromCache,
percent: loaded / size * 100
percent: loaded / size * 100,
...(fromCache == null ? {} : { fromCache })
}
}
}
4 changes: 2 additions & 2 deletions src/upload/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ describe('test upload', () => {
})

test('base Direct.', async () => {
// 文件小于等于 4M 使用直传
// 文件小于 4M 使用直传
const result1 = await observablePromisify(upload(File3M, null, testToken))
expect(result1).toStrictEqual((await mockApi.direct()).data)

Expand All @@ -58,7 +58,7 @@ describe('test upload', () => {
})

test('Resume: base.', async () => {
// 文件大于等于 4M 使用分片
// 文件大于 4M 使用分片
const result = await observablePromisify(upload(File5M, null, testToken))
expect(result).toStrictEqual((await mockApi.uploadComplete()).data)
})
Expand Down
89 changes: 58 additions & 31 deletions src/upload/resume.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import Base, { Progress, UploadInfo, Extra } from './base'

export interface UploadedChunkStorage extends UploadChunkData {
size: number

fromCache?: boolean
}

export interface ChunkLoaded {
Expand Down Expand Up @@ -41,11 +39,34 @@ function isPositiveInteger(n: number) {
}

export default class Resume extends Base {
/**
* @description 文件的分片 chunks
*/
private chunks: Blob[]
/** 当前上传过程中已完成的上传信息 */

/**
* @description 使用缓存的 chunks
*/
private usedCacheList: boolean[]

/**
* @description 来自缓存的上传信息
*/
private cachedUploadedList: UploadedChunkStorage[]

/**
* @description 当前上传过程中已完成的上传信息
*/
private uploadedList: UploadedChunkStorage[]
/** 当前上传片进度信息 */

/**
* @description 当前上传片进度信息
*/
private loaded: ChunkLoaded

/**
* @description 当前上传任务的 id
*/
private uploadId: string

/**
Expand Down Expand Up @@ -98,33 +119,33 @@ export default class Resume extends Base {

private async uploadChunk(chunkInfo: ChunkInfo) {
const { index, chunk } = chunkInfo
const info = this.uploadedList[index]
this.logger.info(`upload part ${index}.`, info)
const cachedInfo = this.cachedUploadedList[index]
this.logger.info(`upload part ${index}, cache:`, cachedInfo)

const shouldCheckMD5 = this.config.checkByMD5
const reuseSaved = () => {
info.fromCache = true
this.usedCacheList[index] = true
this.updateChunkProgress(chunk.size, index)
this.uploadedList[index] = cachedInfo
this.updateLocalCache()
}

// FIXME: 至少判断一下 size
if (info && !shouldCheckMD5) {
if (cachedInfo && !shouldCheckMD5) {
reuseSaved()
return
}

const md5 = await utils.computeMd5(chunk)
this.logger.info('computed part md5.', md5)

if (info && md5 === info.md5) {
if (cachedInfo && md5 === cachedInfo.md5) {
reuseSaved()
return
}

// 有缓存但是没有使用则调整标记
if (info) {
info.fromCache = false
}
// 没有使用缓存设置标记为 false
this.usedCacheList[index] = false

const onProgress = (data: Progress) => {
this.updateChunkProgress(data.loaded, index)
Expand Down Expand Up @@ -158,16 +179,14 @@ export default class Resume extends Base {
size: chunk.size
}

utils.setLocalFileInfo(this.getLocalKey(), {
id: this.uploadId,
data: this.uploadedList
}, this.logger)
this.updateLocalCache()
}

private async mkFileReq() {
const data: UploadChunkBody = {
parts: this.uploadedList.map((value, index) => ({
etag: value.etag,
// 接口要求 index 需要从 1 开始,所以需要整体 + 1
partNumber: index + 1
})),
fname: this.putExtra.fname,
Expand All @@ -193,31 +212,33 @@ export default class Resume extends Base {
}

private async initBeforeUploadChunks() {
const localInfo = utils.getLocalFileInfo(this.getLocalKey(), this.logger)
this.uploadedList = []
this.usedCacheList = []
const cachedInfo = utils.getLocalFileInfo(this.getLocalKey(), this.logger)

// 分片必须和当时使用的 uploadId 配套,所以断点续传需要把本地存储的 uploadId 拿出来
// 假如没有 localInfo 本地信息并重新获取 uploadId
if (!localInfo) {
this.logger.info('resume upload parts from api.')
// 假如没有 cachedInfo 本地信息并重新获取 uploadId
if (!cachedInfo) {
this.logger.info('init upload parts from api.')
const res = await initUploadParts(
this.token,
this.bucketName,
this.key,
this.uploadHost!.getUrl()
)
this.logger.info(`resume upload parts of id: ${res.data.uploadId}.`)
this.logger.info(`initd upload parts of id: ${res.data.uploadId}.`)
this.uploadId = res.data.uploadId
this.uploadedList = []
this.cachedUploadedList = []
} else {
const infoMessage = [
'resume upload parts from local cache',
`total ${localInfo.data.length} part`,
`id is ${localInfo.id}.`
'resume upload parts from local cache,',
`total ${cachedInfo.data.length} part,`,
`id is ${cachedInfo.id}.`
]

this.logger.info(infoMessage.join(', '))
this.uploadedList = localInfo.data
this.uploadId = localInfo.id
this.logger.info(infoMessage.join(' '))
this.cachedUploadedList = cachedInfo.data
this.uploadId = cachedInfo.id
}

this.chunks = utils.getChunks(this.file, this.config.chunkSize)
Expand All @@ -239,6 +260,13 @@ export default class Resume extends Base {
return utils.createLocalKey(this.file.name, this.key, this.file.size)
}

private updateLocalCache() {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

问个问题,如果有 10 个分片,
如果第一次上传,传成功了 5 个分片的时候,然后关了网页
然后再打开页面上传,第 1 个分片上传成功,然后网络请求失败导致其他分片全部失败了
这时候重新上传,是不是 2~5 这几个本来可以复用的分片就废了?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@huangbinjie dart sdk 是怎样的

utils.setLocalFileInfo(this.getLocalKey(), {
id: this.uploadId,
data: this.uploadedList
}, this.logger)
}

private updateChunkProgress(loaded: number, index: number) {
this.loaded.chunks[index] = loaded
this.notifyResumeProgress()
Expand All @@ -257,8 +285,7 @@ export default class Resume extends Base {
this.file.size + 1 // 防止在 complete 未调用的时候进度显示 100%
),
chunks: this.chunks.map((chunk, index) => {
const info = this.uploadedList[index]
const fromCache = info && info.fromCache
const fromCache = this.usedCacheList[index]
return this.getProgressInfoItem(this.loaded.chunks[index], chunk.size, fromCache)
}),
uploadInfo: {
Expand Down