From 1d163250114fd8606b060b8101f6ca48808fa2e4 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 18 Aug 2026 11:38:28 +0200 Subject: [PATCH 1/4] Add debug logging for SQL execution in JobComplexModule --- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py index 6d9a6509a..1b1c478ee 100644 --- a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +++ b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py @@ -3533,7 +3533,10 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci tmp_log.debug(f"bulk execution for {len(arg_list)} jobs") for target_key in sql_key_list: if target_key not in extracted_sqls: + tmp_log.debug(f"no SQLs for {target_key}") continue + else: + tmp_log.debug(f"SQLs for {target_key}: {len(extracted_sqls[target_key]['sqls'])} SQLs") for sql in extracted_sqls[target_key]["sqls"]: self.cur.executemany(sql, extracted_sqls[target_key]["vars"][sql]) # commit From a1a0a0a1e3257ed67668e8708bb1113c59719250 Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 18 Aug 2026 13:35:28 +0200 Subject: [PATCH 2/4] Add debug logging for SQL statements with no bound variables in JobComplexModule --- pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py index 1b1c478ee..ce4c46d8c 100644 --- a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +++ b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py @@ -3538,6 +3538,8 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci else: tmp_log.debug(f"SQLs for {target_key}: {len(extracted_sqls[target_key]['sqls'])} SQLs") for sql in extracted_sqls[target_key]["sqls"]: + if len(extracted_sqls[target_key]["vars"][sql]) == 0: + tmp_log.debug(f"no variables for {target_key} SQL: {sql}") self.cur.executemany(sql, extracted_sqls[target_key]["vars"][sql]) # commit if not self._commit(): From 8f94ea72c8ced8200d0109b1f2d5bac47f3ba86d Mon Sep 17 00:00:00 2001 From: tmaeno Date: Tue, 18 Aug 2026 14:37:31 +0200 Subject: [PATCH 3/4] Add debug logging for SQL statements and variables in JobComplexModule --- .../taskbuffer/db_proxy_mods/job_complex_module.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py index ce4c46d8c..be48f45b8 100644 --- a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +++ b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py @@ -3486,6 +3486,8 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci return_list = [] extracted_sqls = {} es_jobset_map = {} + bad_sql = None + bad_vars = None for args, kwargs, extra_params in arg_list: tmp_extracted_sqls = {} new_kwargs = { @@ -3540,6 +3542,8 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci for sql in extracted_sqls[target_key]["sqls"]: if len(extracted_sqls[target_key]["vars"][sql]) == 0: tmp_log.debug(f"no variables for {target_key} SQL: {sql}") + bad_sql = sql + bad_vars = extracted_sqls[target_key]["vars"][sql] self.cur.executemany(sql, extracted_sqls[target_key]["vars"][sql]) # commit if not self._commit(): @@ -3555,6 +3559,10 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci self._rollback() # error self.dump_error_message(tmp_log) + if bad_sql is not None: + tmp_log.debug(f"bad SQL: {bad_sql}") + if bad_vars is not None: + tmp_log.debug(f"bad variables: {bad_vars}") exec_time = naive_utcnow() - start_time tmp_log.debug("done NG. took %s.%03d sec" % (exec_time.seconds, exec_time.microseconds / 1000)) return False, None, None From 77d1794939962abe0db03f090e7c383df22dd9de Mon Sep 17 00:00:00 2001 From: tmaeno Date: Wed, 19 Aug 2026 10:15:32 +0200 Subject: [PATCH 4/4] fixed get_previous_build_file_spec to have consistent file and dataset even when active jobs are missing --- .../taskbuffer/db_proxy_mods/job_complex_module.py | 13 ------------- .../db_proxy_mods/task_standalone_module.py | 8 +++++--- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py index be48f45b8..6d9a6509a 100644 --- a/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py +++ b/pandaserver/taskbuffer/db_proxy_mods/job_complex_module.py @@ -3486,8 +3486,6 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci return_list = [] extracted_sqls = {} es_jobset_map = {} - bad_sql = None - bad_vars = None for args, kwargs, extra_params in arg_list: tmp_extracted_sqls = {} new_kwargs = { @@ -3535,15 +3533,8 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci tmp_log.debug(f"bulk execution for {len(arg_list)} jobs") for target_key in sql_key_list: if target_key not in extracted_sqls: - tmp_log.debug(f"no SQLs for {target_key}") continue - else: - tmp_log.debug(f"SQLs for {target_key}: {len(extracted_sqls[target_key]['sqls'])} SQLs") for sql in extracted_sqls[target_key]["sqls"]: - if len(extracted_sqls[target_key]["vars"][sql]) == 0: - tmp_log.debug(f"no variables for {target_key} SQL: {sql}") - bad_sql = sql - bad_vars = extracted_sqls[target_key]["vars"][sql] self.cur.executemany(sql, extracted_sqls[target_key]["vars"][sql]) # commit if not self._commit(): @@ -3559,10 +3550,6 @@ def bulk_insert_new_jobs(self, jedi_task_id, arg_list, new_jobset_id_list, speci self._rollback() # error self.dump_error_message(tmp_log) - if bad_sql is not None: - tmp_log.debug(f"bad SQL: {bad_sql}") - if bad_vars is not None: - tmp_log.debug(f"bad variables: {bad_vars}") exec_time = naive_utcnow() - start_time tmp_log.debug("done NG. took %s.%03d sec" % (exec_time.seconds, exec_time.microseconds / 1000)) return False, None, None diff --git a/pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py b/pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py index 7b6b82e36..6aaa5c5c4 100644 --- a/pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py +++ b/pandaserver/taskbuffer/db_proxy_mods/task_standalone_module.py @@ -2038,10 +2038,10 @@ def get_previous_build_file_spec( res_list = self.cur.fetchall() # loop over all datasets for res_item in res_list: - dataset_spec = JediDatasetSpec() - dataset_spec.pack(res_item) + tmp_dataset_spec = JediDatasetSpec() + tmp_dataset_spec.pack(res_item) # get file - var_map = {":jediTaskID": jedi_task_id, ":datasetID": dataset_spec.datasetID, ":type": "lib", ":status1": "failed", ":status2": "cancelled"} + var_map = {":jediTaskID": jedi_task_id, ":datasetID": tmp_dataset_spec.datasetID, ":type": "lib", ":status1": "failed", ":status2": "cancelled"} self.cur.execute(sql_read_file + comment, var_map) res_file_list = self.cur.fetchall() for res_file_item in res_file_list: @@ -2050,6 +2050,7 @@ def get_previous_build_file_spec( tmp_file_spec.pack(res_file_item) if tmp_file_spec.status == "finished": found_flag = True + dataset_spec = tmp_dataset_spec file_spec = tmp_file_spec break # check if the corresponding job is still active @@ -2060,6 +2061,7 @@ def get_previous_build_file_spec( # no active job tmp_log.debug(f"no active job for {tmp_file_spec.lfn} (PandaID={tmp_file_spec.PandaID})") else: + dataset_spec = tmp_dataset_spec file_spec = tmp_file_spec # no more dataset lookup if found_flag: