From d31bd40bff5ad8fd5239166c53f49d19ffc2807c Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Wed, 26 Feb 2025 10:19:38 -0500 Subject: [PATCH 1/3] Implemented processor for BM usage --- process_report/process_report.py | 8 +++- .../processors/bm_usage_processor.py | 20 ++++++++++ .../processors/bu_subsidy_processor.py | 7 +++- .../processors/new_pi_credit_processor.py | 4 ++ .../processors/test_bm_usage_processor.py | 37 +++++++++++++++++++ process_report/tests/util.py | 15 ++++++++ 6 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 process_report/processors/bm_usage_processor.py create mode 100644 process_report/tests/unit/processors/test_bm_usage_processor.py diff --git a/process_report/process_report.py b/process_report/process_report.py index 1610a20..2ed4f63 100644 --- a/process_report/process_report.py +++ b/process_report/process_report.py @@ -25,6 +25,7 @@ new_pi_credit_processor, bu_subsidy_processor, prepayment_processor, + bm_usage_processor, ) ### PI file field names @@ -278,10 +279,15 @@ def main(): ) validate_billable_pi_proc.process() + bm_usage_proc = bm_usage_processor.BMUsageProcessor( + "", invoice_month, validate_billable_pi_proc.data + ) + bm_usage_proc.process() + new_pi_credit_proc = new_pi_credit_processor.NewPICreditProcessor( "", invoice_month, - data=validate_billable_pi_proc.data, + data=bm_usage_proc.data, old_pi_filepath=old_pi_file, initial_credit_amount=new_pi_credit_amount, limit_new_pi_credit_to_partners=( diff --git a/process_report/processors/bm_usage_processor.py b/process_report/processors/bm_usage_processor.py new file mode 100644 index 0000000..ce97205 --- /dev/null +++ b/process_report/processors/bm_usage_processor.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +import pandas + +from process_report.invoices import invoice +from process_report.processors import processor + + +@dataclass +class BMUsageProcessor(processor.Processor): + def _get_bm_project_mask(self): + return pandas.Series(True, index=self.data.index) # TODO: Remove dummy mask + + def _process(self): + bm_projects_mask = self._get_bm_project_mask() + self.data.loc[bm_projects_mask, invoice.PROJECT_FIELD] = self.data.loc[ + bm_projects_mask, invoice.PROJECT_FIELD + ].apply(lambda v: v + " BM Usage") + self.data.loc[bm_projects_mask, invoice.PROJECT_ID_FIELD] = "ESI Bare Metal" + self.data.loc[bm_projects_mask, invoice.INVOICE_EMAIL_FIELD] = "nclinton@bu.edu" diff --git a/process_report/processors/bu_subsidy_processor.py b/process_report/processors/bu_subsidy_processor.py index 8dc35f7..da81827 100644 --- a/process_report/processors/bu_subsidy_processor.py +++ b/process_report/processors/bu_subsidy_processor.py @@ -32,7 +32,12 @@ def _get_subsidy_eligible_projects(data): ] filtered_data = filtered_data[ filtered_data[invoice.INSTITUTION_FIELD] == "Boston University" - ].copy() + ] + filtered_data = ( + filtered_data[ # TODO Does it make sense to test this filter in test cases? + ~(filtered_data[invoice.PROJECT_ID_FIELD] == "ESI Bare Metal") + ] + ) return filtered_data diff --git a/process_report/processors/new_pi_credit_processor.py b/process_report/processors/new_pi_credit_processor.py index 348ce59..51f74ed 100644 --- a/process_report/processors/new_pi_credit_processor.py +++ b/process_report/processors/new_pi_credit_processor.py @@ -87,10 +87,14 @@ def _filter_nonbillables(self, data): def _filter_missing_pis(self, data): return data[~data["Missing PI"]] + def _filter_bm_projects(self, data): + return data[~(data[invoice.PROJECT_ID_FIELD] == "ESI Bare Metal")] + def _get_credit_eligible_projects(self, data: pandas.DataFrame): filtered_data = self._filter_nonbillables(data) filtered_data = self._filter_missing_pis(filtered_data) filtered_data = self._filter_excluded_su_types(filtered_data) + filtered_data = self._filter_bm_projects(filtered_data) if self.limit_new_pi_credit_to_partners: filtered_data = self._filter_partners(filtered_data) diff --git a/process_report/tests/unit/processors/test_bm_usage_processor.py b/process_report/tests/unit/processors/test_bm_usage_processor.py new file mode 100644 index 0000000..54979ef --- /dev/null +++ b/process_report/tests/unit/processors/test_bm_usage_processor.py @@ -0,0 +1,37 @@ +from unittest import TestCase + +import pandas + +from process_report.tests import util as test_utils + + +class TestBUSubsidyProcessor(TestCase): + def test_get_bm_project_mask(self): + test_invoice = pandas.DataFrame({}) + + answer_invoice = test_invoice.iloc[[0, 2]] + + bm_usage_proc = test_utils.new_bm_usage_processor(data=test_invoice) + bm_project_mask = bm_usage_proc._get_bm_project_mask() + self.assertTrue(test_invoice[bm_project_mask].equals(answer_invoice)) + + def test_process_bm_usage(self): + test_invoice = pandas.DataFrame( + { + "Project - Allocation": ["test", "test bm-bm"], + "Project - Allocation ID": [None] * 2, + "Invoice Email": [None] * 2, + } + ) + + answer_invoice = pandas.DataFrame( + { + "Project - Allocation": ["test BM Usage", "test bm-bm BM Usage"], + "Project - Allocation ID": ["ESI Bare Metal"] * 2, + "Invoice Email": ["nclinton@bu.edu"] * 2, + } + ) + + bm_usage_proc = test_utils.new_bm_usage_processor(data=test_invoice) + bm_usage_proc.process() + self.assertTrue(bm_usage_proc.data.equals(answer_invoice)) diff --git a/process_report/tests/util.py b/process_report/tests/util.py index 8a39b40..0fbdddf 100644 --- a/process_report/tests/util.py +++ b/process_report/tests/util.py @@ -14,6 +14,7 @@ new_pi_credit_processor, bu_subsidy_processor, prepayment_processor, + bm_usage_processor, ) @@ -175,3 +176,17 @@ def new_prepayment_processor( prepay_debits_filepath, upload_to_s3, ) + + +def new_bm_usage_processor( + name="", + invoice_month="0000-00", + data=None, +): + if data is None: + data = pandas.DataFrame() + return bm_usage_processor.BMUsageProcessor( + name, + invoice_month, + data, + ) From 3b757cee13e69a008466605e74677a46d1e270b4 Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Wed, 26 Feb 2025 11:16:40 -0500 Subject: [PATCH 2/3] Marked where credits and discount processing begins This is to improve readability, since we would always want processing for credits and discounts to be done after all other processing steps that just adds metadata --- process_report/process_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/process_report/process_report.py b/process_report/process_report.py index 2ed4f63..40b2440 100644 --- a/process_report/process_report.py +++ b/process_report/process_report.py @@ -284,6 +284,8 @@ def main(): ) bm_usage_proc.process() + ### Credits and discounts processing + new_pi_credit_proc = new_pi_credit_processor.NewPICreditProcessor( "", invoice_month, From 811f9bbafd9b3f056b6a8a89a146206f1d8ae0f7 Mon Sep 17 00:00:00 2001 From: Quan Pham Date: Tue, 25 Feb 2025 11:42:51 -0500 Subject: [PATCH 3/3] Implemented processor for baremetal --- process_report/invoices/bm_invoice.py | 30 +++++++++++++++++++++++++++ process_report/process_report.py | 12 +++++++++++ 2 files changed, 42 insertions(+) create mode 100644 process_report/invoices/bm_invoice.py diff --git a/process_report/invoices/bm_invoice.py b/process_report/invoices/bm_invoice.py new file mode 100644 index 0000000..470cd66 --- /dev/null +++ b/process_report/invoices/bm_invoice.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass + + +from process_report.invoices import invoice + + +@dataclass +class BMInvoice(invoice.Invoice): + export_columns_list = [ + invoice.INVOICE_DATE_FIELD, + invoice.PROJECT_FIELD, + invoice.PROJECT_ID_FIELD, + invoice.PI_FIELD, + invoice.INVOICE_EMAIL_FIELD, + invoice.INVOICE_ADDRESS_FIELD, + invoice.INSTITUTION_FIELD, + invoice.INSTITUTION_ID_FIELD, + invoice.SU_HOURS_FIELD, + invoice.SU_TYPE_FIELD, + invoice.RATE_FIELD, + invoice.COST_FIELD, + invoice.CREDIT_FIELD, + invoice.CREDIT_CODE_FIELD, + invoice.BALANCE_FIELD, + ] + + def _prepare_export(self): + self.export_data = self.data[ + self.data[invoice.PROJECT_ID_FIELD] == "ESI Bare Metal" + ] diff --git a/process_report/process_report.py b/process_report/process_report.py index 40b2440..ebb3fa6 100644 --- a/process_report/process_report.py +++ b/process_report/process_report.py @@ -9,6 +9,7 @@ from process_report import util from process_report.invoices import ( + bm_invoice, lenovo_invoice, nonbillable_invoice, billable_invoice, @@ -190,6 +191,12 @@ def main(): default="Lenovo", help="Name of output csv for Lenovo SU Types invoice", ) + parser.add_argument( + "--bm-usage-file", + required=False, + default="bm_usage", + help="Name of output csv for Lenovo SU Types invoice", + ) parser.add_argument( "--old-pi-file", required=False, @@ -366,6 +373,10 @@ def main(): name="", invoice_month=invoice_month, data=processed_data.copy() ) + bm_inv = bm_invoice.BMInvoice( + name=args.bm_usage_file, invoice_month=invoice_month, data=processed_data + ) + util.process_and_export_invoices( [ lenovo_inv, @@ -375,6 +386,7 @@ def main(): bu_internal_inv, pi_inv, moca_prepaid_inv, + bm_inv, ], args.upload_to_s3, )