Batch - remove potential race condition (#3965)

This commit is contained in:
Bert Blommers 2021-05-28 06:55:01 +01:00 committed by GitHub
parent 3265e89fd3
commit 037973c8d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -534,9 +534,9 @@ class Job(threading.Thread, BaseModel, DockerModel):
result = container.wait() result = container.wait()
if self.stop or result["StatusCode"] != 0: if self.stop or result["StatusCode"] != 0:
self.job_state = "FAILED" self._mark_stopped(success=False)
else: else:
self.job_state = "SUCCEEDED" self._mark_stopped(success=True)
except Exception as err: except Exception as err:
logger.error( logger.error(
@ -544,7 +544,7 @@ class Job(threading.Thread, BaseModel, DockerModel):
self.name, err self.name, err
) )
) )
self.job_state = "FAILED" self._mark_stopped(success=False)
container.kill() container.kill()
finally: finally:
container.remove() container.remove()
@ -554,10 +554,14 @@ class Job(threading.Thread, BaseModel, DockerModel):
self.name, err self.name, err
) )
) )
self.job_state = "FAILED" self._mark_stopped(success=False)
def _mark_stopped(self, success=True):
# Ensure that job_stopped/job_stopped_at-attributes are set first
# The describe-method needs them immediately when job_state is set
self.job_stopped = True self.job_stopped = True
self.job_stopped_at = datetime.datetime.now() self.job_stopped_at = datetime.datetime.now()
self.job_state = "SUCCEEDED" if success else "FAILED"
def terminate(self, reason): def terminate(self, reason):
if not self.stop: if not self.stop:
@ -579,9 +583,7 @@ class Job(threading.Thread, BaseModel, DockerModel):
self.name, dependent_job.name self.name, dependent_job.name
) )
) )
self.job_state = "FAILED" self._mark_stopped(success=False)
self.job_stopped = True
self.job_stopped_at = datetime.datetime.now()
return False return False
time.sleep(1) time.sleep(1)