Projects that belong to courses are now non-billable

QuanMPhm · QuanMPhm · commit c9eaad9fbf24 · 2025-11-03T12:10:58.000-05:00
Leveraging the private field `Is Course?` in Coldfront [1], if this field is set to `Yes`, the project/allocation is considered part of a course. For now, any project that is a part of a course and belongs to Boston University is non-billable. [2] If the field is not set, it's assumed to be not part of a course [1] https://github.com/ubccr/coldfront/blob/a4b6b4c1d18535f61de5aa4bbfe509d7d8ca4b0f/coldfront/core/allocation/management/commands/add_allocation_defaults.py#L56 [2] CCI-MOC/non-billable-projects#66 (comment)
diff --git a/process_report/invoices/invoice.py b/process_report/invoices/invoice.py
@@ -56,6 +56,7 @@
 PROJECT_NAME_FIELD = "Project"
 GROUP_MANAGED_FIELD = "MGHPCC Managed"
 CLUSTER_NAME_FIELD = "Cluster Name"
+IS_COURSE_FIELD = "Is Course"
 ###
 
 
diff --git a/process_report/processors/coldfront_fetch_processor.py b/process_report/processors/coldfront_fetch_processor.py
@@ -19,6 +19,7 @@
 CF_ATTR_ALLOCATED_PROJECT_NAME = "Allocated Project Name"
 CF_ATTR_ALLOCATED_PROJECT_ID = "Allocated Project ID"
 CF_ATTR_INSTITUTION_SPECIFIC_CODE = "Institution-Specific Code"
+CF_ATTR_IS_COURSE = "Is Course?"
 
 
 @dataclass
@@ -96,10 +97,15 @@ def _get_allocation_data(self, coldfront_api_data):
                 institute_code = project_dict["attributes"].get(
                     CF_ATTR_INSTITUTION_SPECIFIC_CODE, "N/A"
                 )
+                is_course = (
+                    project_dict["attributes"].get(CF_ATTR_IS_COURSE, "No").lower()
+                    == "yes"
+                )
                 allocation_data[project_id] = {
                     invoice.PROJECT_FIELD: project_name,
                     invoice.PI_FIELD: pi_name,
                     invoice.INSTITUTION_ID_FIELD: institute_code,
+                    invoice.IS_COURSE_FIELD: is_course,
                 }
             except KeyError:
                 continue
@@ -120,13 +126,15 @@ def _validate_allocation_data(self, allocation_data):
             )
 
     def _apply_allocation_data(self, allocation_data):
+        self.data[invoice.IS_COURSE_FIELD] = False
         for project_id, data in allocation_data.items():
             mask = self.data[invoice.PROJECT_ID_FIELD] == project_id
             self.data.loc[mask, invoice.PROJECT_FIELD] = data[invoice.PROJECT_FIELD]
             self.data.loc[mask, invoice.PI_FIELD] = data[invoice.PI_FIELD]
             self.data.loc[mask, invoice.INSTITUTION_ID_FIELD] = data[
                 invoice.INSTITUTION_ID_FIELD
             ]
+            self.data.loc[mask, invoice.IS_COURSE_FIELD] = data[invoice.IS_COURSE_FIELD]
 
     def _process(self):
         api_data = self._get_coldfront_api_data()
diff --git a/process_report/processors/validate_billable_pi_processor.py b/process_report/processors/validate_billable_pi_processor.py
@@ -12,6 +12,7 @@
 
 
 NONBILLABLE_CLUSTERS = ["ocp-test"]
+NONBILLABLE_COURSE_INSTITUTIONS = ["Boston University"]
 
 
 @dataclass
@@ -56,6 +57,10 @@ def _str_to_lowercase(data):
             .apply(_str_to_lowercase)
             .isin(nonbillable_projects_lowercase)
             & ~data[invoice.CLUSTER_NAME_FIELD].isin(NONBILLABLE_CLUSTERS)
+            & ~(
+                data[invoice.IS_COURSE_FIELD]
+                & data[invoice.INSTITUTION_FIELD].isin(NONBILLABLE_COURSE_INSTITUTIONS)
+            )
         )
 
     def _process(self):
diff --git a/process_report/tests/unit/processors/test_coldfront_fetch_processor.py b/process_report/tests/unit/processors/test_coldfront_fetch_processor.py
@@ -13,6 +13,7 @@ def _get_test_invoice(
         pi=None,
         institute_code=None,
         cluster_name=None,
+        is_course=None,
     ):
         if not pi:
             pi = [""] * len(allocation_project_id)
@@ -26,31 +27,40 @@ def _get_test_invoice(
         if not cluster_name:
             cluster_name = [""] * len(allocation_project_id)
 
+        if not is_course:
+            is_course = [None] * len(allocation_project_id)
+
         return pandas.DataFrame(
             {
                 "Manager (PI)": pi,
                 "Project - Allocation": allocation_project_name,
                 "Project - Allocation ID": allocation_project_id,
                 "Institution - Specific Code": institute_code,
                 "Cluster Name": cluster_name,
+                "Is Course": is_course,
             }
         )
 
-    def _get_mock_allocation_data(self, project_id_list, pi_list, institute_code_list):
+    def _get_mock_allocation_data(
+        self, project_id_list, pi_list, institute_code_list, is_course_list=None
+    ):
         mock_data = []
         for i, project in enumerate(project_id_list):
-            mock_data.append(
-                {
-                    "project": {
-                        "pi": pi_list[i],
-                    },
-                    "attributes": {
-                        "Allocated Project ID": project,
-                        "Allocated Project Name": f"{project}-name",
-                        "Institution-Specific Code": institute_code_list[i],
-                    },
-                }
-            )
+            mock_project_dict = {
+                "project": {
+                    "pi": pi_list[i],
+                },
+                "attributes": {
+                    "Allocated Project ID": project,
+                    "Allocated Project Name": f"{project}-name",
+                    "Institution-Specific Code": institute_code_list[i],
+                },
+            }
+
+            if is_course_list:
+                mock_project_dict["attributes"]["Is Course?"] = is_course_list[i]
+
+            mock_data.append(mock_project_dict)
 
         return mock_data
 
@@ -69,6 +79,7 @@ def test_coldfront_fetch(self, mock_get_allocation_data):
             ["P1-name", "P1-name", "P2-name", "P3-name", "P4-name"],
             ["PI1", "PI1", "PI1", "", "PI12"],
             ["IC1", "IC1", "", "", "IC2"],
+            is_course=[False] * 5,
         )
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
             data=test_invoice
@@ -122,6 +133,64 @@ def test_nonbillable_clusters(self, mock_get_allocation_data):
             ["PI1", "PI1", "", ""],
             ["IC1", "IC2", "", ""],
             ["ocp-prod", "stack", "ocp-test", "ocp-test"],
+            [False] * 4,
+        )
+        test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
+            data=test_invoice
+        )
+        test_coldfront_fetch_proc.process()
+        output_invoice = test_coldfront_fetch_proc.data
+        assert output_invoice.equals(answer_invoice)
+
+    @mock.patch(
+        "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
+    )
+    def test_is_course_default_false(self, mock_get_allocation_data):
+        """If 'Is Course?' is not set in the API data, the output 'Is Course' column is False"""
+        mock_get_allocation_data.return_value = self._get_mock_allocation_data(
+            ["P1", "P2"],
+            ["PI1", "PI2"],
+            ["IC1", "IC2"],
+        )
+        test_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"]
+        )  # P3 not in Coldfront API data
+        answer_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"],
+            ["P1-name", "P2-name", ""],
+            ["PI1", "PI2", ""],
+            ["IC1", "IC2", ""],
+            ["", "", ""],
+            [False, False, False],
+        )
+        test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
+            data=test_invoice, nonbillable_projects=["P3"]
+        )
+        test_coldfront_fetch_proc.process()
+        output_invoice = test_coldfront_fetch_proc.data
+        print(output_invoice)
+        print(answer_invoice)
+        assert output_invoice.equals(answer_invoice)
+
+    @mock.patch(
+        "process_report.processors.coldfront_fetch_processor.ColdfrontFetchProcessor._fetch_coldfront_allocation_api",
+    )
+    def test_is_course_values(self, mock_get_allocation_data):
+        """If 'Is Course?' is set in the API data, the output 'Is Course' column reflects True/False"""
+        mock_get_allocation_data.return_value = self._get_mock_allocation_data(
+            ["P1", "P2", "P3"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            is_course_list=["Yes", "No", "yes"],
+        )
+        test_invoice = self._get_test_invoice(["P1", "P2", "P3"])
+        answer_invoice = self._get_test_invoice(
+            ["P1", "P2", "P3"],
+            ["P1-name", "P2-name", "P3-name"],
+            ["PI1", "PI2", "PI3"],
+            ["IC1", "IC2", "IC3"],
+            ["", "", ""],
+            [True, False, True],
         )
         test_coldfront_fetch_proc = test_utils.new_coldfront_fetch_processor(
             data=test_invoice
diff --git a/process_report/tests/unit/processors/test_validate_billable_pi_processor.py b/process_report/tests/unit/processors/test_validate_billable_pi_processor.py
@@ -12,6 +12,8 @@ def test_remove_nonbillables(self):
         projects = [uuid.uuid4().hex for _ in range(10)]
         cluster_names = [uuid.uuid4().hex for _ in range(10)]
         cluster_names[6:8] = ["ocp-test"] * 2  # Test that ocp-test is not billable
+        institutions = ["Test University"] * len(pis)
+        is_course = [False] * len(pis)
         nonbillable_pis = pis[:3]
         nonbillable_projects = [
             project.upper() for project in projects[7:]
@@ -23,6 +25,8 @@ def test_remove_nonbillables(self):
                 "Manager (PI)": pis,
                 "Project - Allocation": projects,
                 "Cluster Name": cluster_names,
+                "Is Course": is_course,
+                "Institution": institutions,
             }
         )
 
@@ -51,6 +55,8 @@ def test_empty_pi_name(self):
                     "ProjectE",
                 ],
                 "Cluster Name": ["test-cluster"] * 5,
+                "Institution": ["Test University"] * 5,
+                "Is Course": [False] * 5,
             }
         )
         assert len(test_data[pandas.isna(test_data["Manager (PI)"])]) == 1
@@ -61,3 +67,32 @@ def test_empty_pi_name(self):
         output_data = validate_billable_pi_proc.data
         output_data = output_data[~output_data["Missing PI"]]
         assert len(output_data[pandas.isna(output_data["Manager (PI)"])]) == 0
+
+    def test_is_course_marks_nonbillable(self):
+        """Rows with Is Course == True should be marked nonbillable; False should be billable."""
+        pis = [uuid.uuid4().hex for _ in range(4)]
+        projects = [uuid.uuid4().hex for _ in range(4)]
+        cluster_names = ["test-cluster"] * 4
+        # Only first project should be nonbillable
+        is_course = [True, False, True, False]
+        institutions = ["Boston University", "Boston University", "test", "test"]
+
+        test_data = pandas.DataFrame(
+            {
+                "Manager (PI)": pis,
+                "Project - Allocation": projects,
+                "Cluster Name": cluster_names,
+                "Is Course": is_course,
+                "Institution": institutions,
+            }
+        )
+
+        validate_proc = test_utils.new_validate_billable_pi_processor(
+            data=test_data, nonbillable_pis=[], nonbillable_projects=[]
+        )
+        validate_proc.process()
+        output = validate_proc.data
+
+        expected_billable = [False, True, True, True]
+        actual_billable = output["Is Billable"].tolist()
+        assert actual_billable == expected_billable
