Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add try/except on release of work Unit and add force to workunit reaper #15129

Open
wants to merge 3 commits into
base: devel
Choose a base branch
from
Open

Add try/except on release of work Unit and add force to workunit reaper #15129

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a39e8d7
add try/ecept on release of work Unit and add force to workunit reaper
tanganellilore Apr 23, 2024
5a0f566
rollback receptor and force release on system
tanganellilore May 2, 2024
926cc8a
add try except also on force release
tanganellilore May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion awx/main/tasks/receptor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ def administrative_workunit_reaper(work_list=None):
- worker_cleanup
These should ordinarily be released when the method finishes, but this is a
cleanup of last-resort, in case something went awry
Release will be forced to be sure that everything will be relased.
"""
receptor_ctl = get_receptor_ctl()
if work_list is None:
Expand All @@ -219,7 +220,7 @@ def administrative_workunit_reaper(work_list=None):
if work_data.get('StateName') in RECEPTOR_ACTIVE_STATES:
continue # do not want to touch active work units
logger.info(f'Reaping orphaned work unit {unit_id} with params {params}')
receptor_ctl.simple_command(f"work release {unit_id}")
receptor_ctl.simple_command(f"work force-release {unit_id}")


class RemoteJobError(RuntimeError):
Expand Down
9 changes: 7 additions & 2 deletions awx/main/tasks/system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -684,8 +684,13 @@ def awx_receptor_workunit_reaper():
jobs_with_unreleased_receptor_units = UnifiedJob.objects.filter(work_unit_id__in=unit_ids).exclude(status__in=ACTIVE_STATES)
for job in jobs_with_unreleased_receptor_units:
logger.debug(f"{job.log_format} is not active, reaping receptor work unit {job.work_unit_id}")
receptor_ctl.simple_command(f"work cancel {job.work_unit_id}")
receptor_ctl.simple_command(f"work release {job.work_unit_id}")
try:
receptor_ctl.simple_command(f"work cancel {job.work_unit_id}")
receptor_ctl.simple_command(f"work release {job.work_unit_id}")
except Exception as e:
# leave the cleaning of work unit to administrative_workunit_reaper
tanganellilore marked this conversation as resolved.
Show resolved Hide resolved
# show only an error on log side to track it
logger.error(f"Error on cancel or release {job.work_unit_id} with error {str(e)}.")

administrative_workunit_reaper(receptor_work_list)

Expand Down