Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Delete failed jobs on db cleanup #70

Merged
merged 1 commit into from
Apr 2, 2024
Merged

Delete failed jobs on db cleanup #70

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Delete failed jobs on db cleanup
  • Loading branch information
@yankovs
yankovs committed Mar 28, 2024
commit 7cc91ecf1b4595f8671a6d575fc9670d83f506e3
15 changes: 14 additions & 1 deletion mcrit/Worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,11 +189,15 @@ def doDbCleanup(self, progress_reporter=NoProgressReporter()):
LOGGER.info(f"Fetching data from the queues.")
unmapped_finished = self.getQueueData(0, 0, method="getMatchesForUnmappedBinary", state="finished")
mapped_finished = self.getQueueData(0, 0, method="getMatchesForMappedBinary", state="finished")
unmapped_failed = self.getQueueData(0, 0, method="getMatchesForUnmappedBinary", state="failed")
mapped_failed = self.getQueueData(0, 0, method="getMatchesForMappedBinary", state="failed")
smda_finished = self.getQueueData(0, 0, method="getMatchesForSmdaReport", state="finished")
recent_samples = set([])
samples_to_be_deleted = {}
jobs_to_be_deleted = []
LOGGER.info(f"Fetching data from the queues, iterating {len(unmapped_finished) + len(unmapped_finished) + len(smda_finished)} items.")
LOGGER.info(
f"Fetching data from the queues, iterating {len(unmapped_finished) + len(unmapped_finished) + len(unmapped_failed) + len(mapped_failed) + len(smda_finished)} items."
)
for job_collection in [unmapped_finished, mapped_finished]:
for job_dict in job_collection:
job = Job(job_dict, None)
Expand All @@ -206,6 +210,15 @@ def doDbCleanup(self, progress_reporter=NoProgressReporter()):
sample_entry = self._storage.getSampleBySha256(job.sha256, is_query=True)
if sample_entry:
samples_to_be_deleted[job.sha256] = sample_entry
for failed_job_collection in [unmapped_failed, mapped_failed]:
for failed_job_dict in failed_job_collection:
job = Job(job_dict, None)
# failed job doesn't have finished_at, delete anyway
jobs_to_be_deleted.append(job)
if job.sha256 not in samples_to_be_deleted:
sample_entry = self._storage.getSampleBySha256(job.sha256, is_query=True)
if sample_entry:
samples_to_be_deleted[job.sha256] = sample_entry
LOGGER.info(f"Decoding SMDA reports for SHA256 hashes.")
for job_dict in smda_finished:
job = Job(job_dict, None)
Expand Down
Loading