Optimize secondary task recollection #2800
Conversation
Signed-off-by: Andrew Brain <[email protected]>
Signed-off-by: Andrew Brain <[email protected]>
Signed-off-by: Andrew Brain <[email protected]>
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
| @@ -1,7 +1,7 @@ | |||
| import sqlalchemy as s | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
C0114: Missing module docstring (missing-module-docstring)
| @@ -1,7 +1,7 @@ | |||
| from celery import chain | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
C0114: Missing module docstring (missing-module-docstring)
| @@ -1,7 +1,7 @@ | |||
| from celery import chain | |||
| import logging | |||
|
|
|||
| def machine_learning_phase(repo_git): | |||
| def machine_learning_phase(repo_git, full_collection): | |||
| from augur.tasks.data_analysis.clustering_worker.tasks import clustering_task | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
C0415: Import outside toplevel (augur.tasks.data_analysis.clustering_worker.tasks.clustering_task) (import-outside-toplevel)
| @@ -1,7 +1,7 @@ | |||
| from celery import chain | |||
| import logging | |||
|
|
|||
| def machine_learning_phase(repo_git): | |||
| def machine_learning_phase(repo_git, full_collection): | |||
| from augur.tasks.data_analysis.clustering_worker.tasks import clustering_task | |||
| from augur.tasks.data_analysis.discourse_analysis.tasks import discourse_analysis_task | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
C0415: Import outside toplevel (augur.tasks.data_analysis.discourse_analysis.tasks.discourse_analysis_task) (import-outside-toplevel)
| @@ -1,7 +1,7 @@ | |||
| from celery import chain | |||
| import logging | |||
|
|
|||
| def machine_learning_phase(repo_git): | |||
| def machine_learning_phase(repo_git, full_collection): | |||
| from augur.tasks.data_analysis.clustering_worker.tasks import clustering_task | |||
| from augur.tasks.data_analysis.discourse_analysis.tasks import discourse_analysis_task | |||
| from augur.tasks.data_analysis.insight_worker.tasks import insight_task | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
C0415: Import outside toplevel (augur.tasks.data_analysis.insight_worker.tasks.insight_task) (import-outside-toplevel)
| @@ -166,7 +166,7 @@ def build_primary_repo_collect_request(session,enabled_phase_names, days_until_c | |||
| primary_gitlab_enabled_phases.append(primary_repo_collect_phase_gitlab) | |||
|
|
|||
| #task success is scheduled no matter what the config says. | |||
| def core_task_success_util_gen(repo_git): | |||
| def core_task_success_util_gen(repo_git, full_collection): | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0613: Unused argument 'full_collection' (unused-argument)
| @@ -186,7 +186,7 @@ def build_secondary_repo_collect_request(session,enabled_phase_names, days_until | |||
|
|
|||
| secondary_enabled_phases.append(secondary_repo_collect_phase) | |||
|
|
|||
| def secondary_task_success_util_gen(repo_git): | |||
| def secondary_task_success_util_gen(repo_git, full_collection): | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0613: Unused argument 'full_collection' (unused-argument)
| @@ -202,12 +202,12 @@ def build_facade_repo_collect_request(session,enabled_phase_names, days_until_co | |||
|
|
|||
| facade_enabled_phases.append(facade_phase) | |||
|
|
|||
| def facade_task_success_util_gen(repo_git): | |||
| def facade_task_success_util_gen(repo_git, full_collection): | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0613: Unused argument 'full_collection' (unused-argument)
| return facade_task_success_util.si(repo_git) | ||
|
|
||
| facade_enabled_phases.append(facade_task_success_util_gen) | ||
|
|
||
| def facade_task_update_weight_util_gen(repo_git): | ||
| def facade_task_update_weight_util_gen(repo_git, full_collection): |
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0613: Unused argument 'full_collection' (unused-argument)
| @@ -222,7 +222,7 @@ def build_ml_repo_collect_request(session,enabled_phase_names, days_until_collec | |||
|
|
|||
| ml_enabled_phases.append(machine_learning_phase) | |||
|
|
|||
| def ml_task_success_util_gen(repo_git): | |||
| def ml_task_success_util_gen(repo_git, full_collection): | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0613: Unused argument 'full_collection' (unused-argument)
Description
Overall
pull_request_files,pull_request_commits, andpull_request_reviewswhen they are recollectedImplementation Details
full_collectionboolean flag sostart_data_collectionknows what to passAugurTaskRoutine.start_data_collectioniterate through the tuples and pass therepo_gitandfull_collectionflag to each phasefull_collectionflag to every phase's method arguments. This had to be done sinceAugurTaskRoutine.start_data_collectionpasses the same arguments for every phaseget_secondary_data_last_collectedand get prs that have been updated since a datepull_request_filesto only get pr numbers for updated prs iffull_collectionflag is falsepull_request_commitsto only get pr urls for updated prs iffull_collectionflag is falsepull_request_reviewsto only get pr numbers for updated prs iffull_collectionflag is falseNotes for Reviewers
I have not tested this yet, I will change it from a draft pr to a pr when it is test
Signed commits