[DEV] correct logging and various bug

Author: vincent-laurent

examples/classification.ipynb

@@ -110,9 +110,12 @@ def compute_metrics(self, metric: dict):
         from palma import logger
         for name, fun in metric.items():
             self._compute_metric(name, fun)
-        logger.logger.log_metrics(
-            {k: str(v) for k, v in self.get_test_metrics().to_dict().items()},
-            path="metrics")
+
+        for m_name, metric_fold in self.get_test_metrics().to_dict().items():
+            for k, v in metric_fold.items():
+                if isinstance(v, float) or isinstance(v, int):
+                    logger.logger.log_metrics(
+                        {f"{m_name}_fold{k}": v}, path="metrics")
 
     def _compute_metric(self, name: str, fun: typing.Callable):
         """

palma/components/performance.py

@@ -72,29 +72,6 @@ def learning_data(classification_project, classification_data):
     return classification_project, learn, X, y
 
 
-@pytest.fixture(scope='module')
-def get_scoring_analyser(learning_data):
-    project, model, X, y = learning_data
-    perf = performance.ScoringAnalysis(on="indexes_train_test")
-    perf._add(project, model)
-
-    perf.compute_metrics(metric={
-        metrics.roc_auc_score.__name__: metrics.roc_auc_score,
-        metrics.roc_curve.__name__: metrics.roc_curve
-    })
-    return perf
-
-
-@pytest.fixture(scope='module')
-def get_shap_analyser(learning_data):
-    project, model, X, y = learning_data
-    perf = performance.ShapAnalysis(on="indexes_val", n_shap=100,
-                                    compute_interaction=True)
-    perf(project, model)
-
-    return perf
-
-
 @pytest.fixture(scope='module')
 def learning_data_regression(regression_data):
     from palma import set_logger
@@ -129,9 +106,6 @@ def get_regression_analyser(learning_data_regression):
     return perf
 
 
-
-
-
 @pytest.fixture(scope='module')
 def build_classification_project(unbuilt_classification_project,
                                  classification_data):

tests/conftest.py

@@ -146,4 +146,3 @@ def test_artifact_logging():
     logger.logger.log_metrics({'a': 1}, "metric")
     logger.logger.log_artifact(fig, "figure")
 
-

tests/test_component/test_logger.py

@@ -9,14 +9,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import matplotlib
+import numpy as np
 import pytest
-from sklearn import metrics
+import pandas as pd
+from sklearn import metrics, model_selection
 
-from palma.components import performance
+from palma.components import performance, FileSystemLogger, MLFlowLogger
+from sklearn.ensemble import RandomForestClassifier
+import tempfile
+from palma import ModelEvaluation, Project
+from palma import set_logger
 
 matplotlib.use("agg")
 
 
+@pytest.fixture(scope='module')
+def get_scoring_analyser(classification_data):
+    set_logger(FileSystemLogger(tempfile.gettempdir() + "/logger"))
+
+    X, y = classification_data
+    X = pd.DataFrame(X)
+    y = pd.Series(y)
+    project = Project(problem="classification",
+                      project_name=str(np.random.uniform()))
+
+    project.start(
+        X, y,
+        splitter=model_selection.ShuffleSplit(n_splits=4, random_state=42))
+    estimator = RandomForestClassifier()
+
+    learn = ModelEvaluation(estimator)
+    learn.fit(project)
+
+    perf = performance.ScoringAnalysis(on="indexes_val")
+    perf(project, learn)
+
+    perf.compute_metrics(metric={
+        metrics.roc_auc_score.__name__: metrics.roc_auc_score,
+        metrics.roc_curve.__name__: metrics.roc_curve
+    })
+    return perf
+
+
+@pytest.fixture(scope='module')
+def get_shap_analyser(learning_data):
+    project, model, X, y = learning_data
+    perf = performance.ShapAnalysis(on="indexes_val", n_shap=100,
+                                    compute_interaction=True)
+
+    perf(project, model)
+
+    return perf
+
+
 def test_classification_perf(get_scoring_analyser):
     performance.plot.figure(figsize=(6, 6), dpi=200)
     get_scoring_analyser.plot_roc_curve(