From cf064556f5f32a8529db322de37cc0a9b51e3970 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Tue, 20 Oct 2020 02:02:53 +0200 Subject: [PATCH 1/8] fix(Gitlab): importe opened issues only --- .../issue/providers/gitlab/gitlab-api/gitlab-api.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts index fff3415646..3017a0590b 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts @@ -86,7 +86,7 @@ export class GitlabApiService { private _getProjectIssues$(pageNumber: number, cfg: GitlabCfg): Observable { return this._sendRequest$({ - url: `${this.apiLink(cfg)}/issues?order_by=updated_at&per_page=100&page=${pageNumber}` + url: `${this.apiLink(cfg)}/issues?state=opened&order_by=updated_at&per_page=100&page=${pageNumber}` }, cfg).pipe( take(1), map((issues: GitlabOriginalIssue[]) => { From 010fbd84adc70324ec27fe9a1b1f6162fc4e7776 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Tue, 20 Oct 2020 02:07:51 +0200 Subject: [PATCH 2/8] fix(Gitlab): improve search feature --- .../gitlab/gitlab-api/gitlab-api.service.ts | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts index 3017a0590b..299761e535 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts @@ -61,27 +61,29 @@ export class GitlabApiService { } searchIssueInProject$(searchText: string, cfg: GitlabCfg): Observable { - const filterFn = (issue: GitlabIssue) => { - try { - return issue.title.toLowerCase().match(searchText.toLowerCase()) - || issue.body.toLowerCase().match(searchText.toLowerCase()); - } catch (e) { - console.warn('RegEx Error', e); - return false; - } - }; if (!this._isValidSettings(cfg)) { return EMPTY; } - return this.getProjectData$(cfg) - .pipe( - // a single request should suffice - share(), - map((issues: GitlabIssue[]) => - issues.filter(filterFn) - .map(mapGitlabIssueToSearchResult) - ), - ); + return this._sendRequest$({ + url: `${this.apiLink(cfg)}/issues?search=${searchText}&order_by=updated_at` + }, cfg).pipe( + map((issues: GitlabOriginalIssue[]) => { + return issues ? issues.map(mapGitlabIssue) : []; + }), + mergeMap( + (issues: GitlabIssue[]) => { + if (issues && issues.length) { + return forkJoin([ + ...issues.map(issue => this.getIssueWithComments$(issue, cfg)) + ]); + } else { + return of([]); + } + }), + map((issues: GitlabIssue[]) => { + return issues ? issues.map(mapGitlabIssueToSearchResult) : []; + }) + ); } private _getProjectIssues$(pageNumber: number, cfg: GitlabCfg): Observable { From 2b43d012163079f0ff3400c48dafc786512f2dde Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Tue, 20 Oct 2020 02:09:58 +0200 Subject: [PATCH 3/8] removing unused import --- .../issue/providers/gitlab/gitlab-api/gitlab-api.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts index 299761e535..c46f440f70 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts @@ -8,7 +8,7 @@ import { GitlabOriginalComment, GitlabOriginalIssue } from './gitlab-api-respons import { HANDLED_ERROR_PROP_STR } from 'src/app/app.constants'; import { GITLAB_API_BASE_URL, GITLAB_URL_REGEX, GITLAB_PROJECT_REGEX } from '../gitlab.const'; import { T } from 'src/app/t.const'; -import { catchError, filter, map, mergeMap, share, switchMap, take } from 'rxjs/operators'; +import { catchError, filter, map, mergeMap, switchMap, take } from 'rxjs/operators'; import { GitlabIssue } from '../gitlab-issue/gitlab-issue.model'; import { mapGitlabIssue, mapGitlabIssueToSearchResult } from '../gitlab-issue/gitlab-issue-map.util'; import { SearchResultItem } from '../../../issue.model'; From e400189d1bc5140f43ca4d22e8496741cd81b883 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Tue, 20 Oct 2020 02:19:19 +0200 Subject: [PATCH 4/8] fix(Gitlab): send less requests to update issues --- .../gitlab/gitlab-api/gitlab-api.service.ts | 14 +++++++++----- .../gitlab/gitlab-issue/gitlab-issue.effects.ts | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts index c46f440f70..a2cdd9fea4 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts @@ -8,7 +8,7 @@ import { GitlabOriginalComment, GitlabOriginalIssue } from './gitlab-api-respons import { HANDLED_ERROR_PROP_STR } from 'src/app/app.constants'; import { GITLAB_API_BASE_URL, GITLAB_URL_REGEX, GITLAB_PROJECT_REGEX } from '../gitlab.const'; import { T } from 'src/app/t.const'; -import { catchError, filter, map, mergeMap, switchMap, take } from 'rxjs/operators'; +import { catchError, filter, map, mergeMap, take } from 'rxjs/operators'; import { GitlabIssue } from '../gitlab-issue/gitlab-issue.model'; import { mapGitlabIssue, mapGitlabIssueToSearchResult } from '../gitlab-issue/gitlab-issue-map.util'; import { SearchResultItem } from '../../../issue.model'; @@ -42,10 +42,14 @@ export class GitlabApiService { } getById$(id: number, cfg: GitlabCfg): Observable { - return this.getProjectData$(cfg) - .pipe(switchMap(issues => { - return issues.filter(issue => issue.id === id); - })); + return this._sendRequest$({ + url: `${this.apiLink(cfg)}/issues/${id}` + }, cfg).pipe( + mergeMap( + (issue: GitlabOriginalIssue) => { + return this.getIssueWithComments$(mapGitlabIssue(issue), cfg); + }) + ); } getIssueWithComments$(issue: GitlabIssue, cfg: GitlabCfg): Observable { diff --git a/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts b/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts index 3ce285f019..a04efacce7 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts @@ -38,7 +38,7 @@ export class GitlabIssueEffects { ) .map(({task}: { cfg: GitlabCfg, task: TaskWithSubTasks }) => task) ), - tap((gitlabTasks: TaskWithSubTasks[]) => gitlabTasks.forEach((gitlabTask) => { + tap((gitlabTasks: TaskWithSubTasks[]) => { if (gitlabTasks && gitlabTasks.length > 0) { this._snackService.open({ msg: T.F.GITLAB.S.POLLING, @@ -48,7 +48,7 @@ export class GitlabIssueEffects { gitlabTasks.forEach((task) => this._issueService.refreshIssue(task, true, false)); } - })), + }), ); private _pollTimer$: Observable = timer(GITLAB_INITIAL_POLL_DELAY, GITLAB_POLL_INTERVAL); From 223dfd45aeb09ee9db29030ae0d1be35c5aa0ee3 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Thu, 22 Oct 2020 10:29:05 +0200 Subject: [PATCH 5/8] refactor(issue): add optional refreshIssues method Adding optional refreshIssues method to issue service interface Allow for future implementation from issue providers --- .../features/issue/issue-service-interface.ts | 8 ++++ src/app/features/issue/issue.service.ts | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/app/features/issue/issue-service-interface.ts b/src/app/features/issue/issue-service-interface.ts index 40a542a478..0094a8d10e 100644 --- a/src/app/features/issue/issue-service-interface.ts +++ b/src/app/features/issue/issue-service-interface.ts @@ -19,5 +19,13 @@ export interface IssueServiceInterface { issue: IssueData } | null>; + refreshIssues?(tasks: Task[], + isNotifySuccess: boolean, + isNotifyNoUpdateRequired: boolean): Promise<{ + task: Task, + taskChanges: Partial, + issue: IssueData + }[]>; + getMappedAttachments?(issueDataIN: IssueData): TaskAttachment[]; } diff --git a/src/app/features/issue/issue.service.ts b/src/app/features/issue/issue.service.ts index dce52466ac..5b550219a9 100644 --- a/src/app/features/issue/issue.service.ts +++ b/src/app/features/issue/issue.service.ts @@ -96,6 +96,50 @@ export class IssueService { } } + async refreshIssues( + tasks: Task[], + isNotifySuccess: boolean = true, + isNotifyNoUpdateRequired: boolean = false + ): Promise { + // dynamic map that has a list of tasks for every entry where the entry is an issue type + const tasksIssueIdsByIssueType: any = {}; + const tasksWithoutIssueId = []; + const tasksWithoutMethod = []; + + for (const task of tasks) { + if (!task.issueId || !task.issueType) { + tasksWithoutIssueId.push(task); + } else if (!this.ISSUE_SERVICE_MAP[task.issueType].refreshIssues) { + tasksWithoutMethod.push(task); + } + else if (!tasksIssueIdsByIssueType[task.issueType]){ + tasksIssueIdsByIssueType[task.issueType] = []; + tasksIssueIdsByIssueType[task.issueType].push(task); + } else { + tasksIssueIdsByIssueType[task.issueType].push(task); + } + } + + for (const issuesType of Object.keys(tasksIssueIdsByIssueType)) { + const updates = await (this.ISSUE_SERVICE_MAP[issuesType].refreshIssues as any)(tasksIssueIdsByIssueType[issuesType], isNotifySuccess, isNotifyNoUpdateRequired); + if (updates) { + for (const update of updates) { + if (this.ISSUE_REFRESH_MAP[issuesType][update.task.issueId]) { + this.ISSUE_REFRESH_MAP[issuesType][update.task.issueId].next(update.issue); + } + this._taskService.update(update.task.id, update.taskChanges); + } + } + } + + for (const taskWithoutIssueId of tasksWithoutIssueId) { + throw new Error('No issue task ' + taskWithoutIssueId.id); + } + for (const taskWithoutMethod of tasksWithoutMethod) { + throw new Error('Issue method not available ' + taskWithoutMethod); + } + } + async addTaskWithIssue( issueType: IssueProviderKey, issueIdOrData: string | number | IssueDataReduced, From 094c35d4abd465fee5560691e2dd06bfd1db52a7 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Thu, 22 Oct 2020 10:38:25 +0200 Subject: [PATCH 6/8] refactor(Gitlab): add method to update list of issues --- .../gitlab/gitlab-api/gitlab-api.service.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts index a2cdd9fea4..568e332ee1 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-api/gitlab-api.service.ts @@ -52,6 +52,33 @@ export class GitlabApiService { ); } + getByIds$(ids: string[], cfg: GitlabCfg): Observable { + let queryParams = 'iids[]='; + for (let i = 0; i < ids.length ; i++) { + if (i === ids.length - 1) { + queryParams += ids[i]; + } else { + queryParams += `${ids[i]}&iids[]=`; + } + } + return this._sendRequest$({ + url: `${this.apiLink(cfg)}/issues?${queryParams}&per_page=100` + }, cfg).pipe( + map((issues: GitlabOriginalIssue[]) => { + return issues ? issues.map(mapGitlabIssue) : []; + }), + mergeMap((issues: GitlabIssue[]) => { + if (issues && issues.length) { + return forkJoin([ + ...issues.map(issue => this.getIssueWithComments$(issue, cfg)) + ]); + } else { + return of([]); + } + }) + ); + } + getIssueWithComments$(issue: GitlabIssue, cfg: GitlabCfg): Observable { return this._getIssueComments$(issue.id, 1, cfg).pipe( map((comments) => { From f998429f0eb0a10a209af477cac82827e7cf7475 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Thu, 22 Oct 2020 10:39:34 +0200 Subject: [PATCH 7/8] feat(Gitlab): Bulk update issues --- .../gitlab-common-interfaces.service.ts | 72 +++++++++++++++++++ .../gitlab-issue/gitlab-issue.effects.ts | 3 +- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts b/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts index 7b3fe067a9..e4995b3164 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts @@ -108,6 +108,78 @@ export class GitlabCommonInterfacesService implements IssueServiceInterface { return null; } + async refreshIssues( + tasks: Task[], + isNotifySuccess: boolean = true, + isNotifyNoUpdateRequired: boolean = false + ): Promise<{ task: Task, taskChanges: Partial, issue: GitlabIssue }[]> { + // First sort the tasks by the issueId + // because the API returns it in a desc order by issue iid(issueId) + // so it makes the update check easier and faster + tasks.sort((a, b) => +(b.issueId as string) - +(a.issueId as string)); + const projectId = tasks && tasks[0].projectId ? tasks[0].projectId : 0; + if (!projectId) { + throw new Error('No projectId'); + } + + const cfg = await this._getCfgOnce$(projectId).toPromise(); + const issues: GitlabIssue[] = []; + const paramsCount = 59; // Can't send more than 59 issue id For some reason it returns 502 bad gateway + let ids; + let i = 0; + while (i < tasks.length) { + ids = []; + for (let j = 0; j < paramsCount && i < tasks.length ; j++, i++) { + ids.push(tasks[i].issueId); + } + issues.push(...(await this._gitlabApiService.getByIds$(ids as string[], cfg).toPromise())); + } + + const updatedIssues: { task: Task, taskChanges: Partial, issue: GitlabIssue }[] = []; + + for (i = 0; i < tasks.length; i++) { + const issueUpdate: number = new Date(issues[i].updated_at).getTime(); + const commentsByOthers = (cfg.filterUsername && cfg.filterUsername.length > 1) + ? issues[i].comments.filter(comment => comment.author.username !== cfg.filterUsername) + : issues[i].comments; + + const updates: number[] = [ + ...(commentsByOthers.map(comment => new Date(comment.created_at).getTime())), + issueUpdate + ].sort(); + const lastRemoteUpdate = updates[updates.length - 1]; + const wasUpdated = lastRemoteUpdate > (tasks[i].issueLastUpdated || 0); + + if (wasUpdated) { + updatedIssues.push({ + task: tasks[i], + taskChanges: { + issueWasUpdated: true, + issueLastUpdated: lastRemoteUpdate, + title: this._formatIssueTitle(issues[i].number, issues[i].title), + }, + issue: issues[i], + }); + } + + if (wasUpdated && isNotifySuccess) { + this._snackService.open({ + ico: 'cloud_download', + translateParams: { + issueText: this._formatIssueTitleForSnack(issues[i].number, issues[i].title) + }, + msg: T.F.GITLAB.S.ISSUE_UPDATE, + }); + } else if (isNotifyNoUpdateRequired) { + this._snackService.open({ + msg: T.F.GITLAB.S.ISSUE_NO_UPDATE_REQUIRED, + ico: 'cloud_download', + }); + } + } + return updatedIssues; + } + getAddTaskData(issue: GitlabIssue): { title: string; additionalFields: Partial } { return { title: this._formatIssueTitle(issue.number, issue.title), diff --git a/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts b/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts index a04efacce7..f8faf34217 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-issue/gitlab-issue.effects.ts @@ -45,8 +45,7 @@ export class GitlabIssueEffects { svgIco: 'gitlab', isSpinner: true, }); - gitlabTasks.forEach((task) => - this._issueService.refreshIssue(task, true, false)); + this._issueService.refreshIssues(gitlabTasks, true, false); } }), ); From ab1aef74978a8d2d2938ddaf3b4bc6a296ef2f52 Mon Sep 17 00:00:00 2001 From: "mostafa.amen07" Date: Thu, 22 Oct 2020 10:42:19 +0200 Subject: [PATCH 8/8] refactor(Gitlab): remove a hard coded part --- .../issue/providers/gitlab/gitlab-common-interfaces.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts b/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts index e4995b3164..bfb16919e6 100644 --- a/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts +++ b/src/app/features/issue/providers/gitlab/gitlab-common-interfaces.service.ts @@ -100,7 +100,7 @@ export class GitlabCommonInterfacesService implements IssueServiceInterface { taskChanges: { issueWasUpdated: true, issueLastUpdated: lastRemoteUpdate, - title: `#${issue.number} ${issue.title}`, + title: this._formatIssueTitle(issue.number, issue.title), }, issue, };