Merge branch 'main' into main-public

Author: ashahba

GetStarted.md

@@ -61,7 +61,8 @@ approaches.
 
 3. **Install Intel Transfer Learning Tool**
 
-   Use the Basic Installation instructions unless you plan on making code changes.
+   Use the Basic Installation instructions unless you plan on making code changes or installing the latest code from the repository.
+   Please note that mixing basic and advanced installation options within the same virtual environment is not supported.
 
    a. **Basic Installation**
 
@@ -107,29 +108,6 @@ approaches.
    tlt --help
    ```
 
-6. **Prepare the Dataset**
-
-   The Intel Transfer Learning Tool can use datasets from existing dataset catalogs
-   or custom datasets that you have on your machine.  The following CLI and API
-   examples use the Intel Transfer Learning Tool's custom dataset option
-   (`--dataset-dir`) with the TensorFlow flowers dataset.
-
-   ```
-   # Create a directory for the dataset to be downloaded
-   DATASET_DIR=/tmp/dataset
-   mkdir -p ${DATASET_DIR}
-
-   # Download and extract the dataset (be sure https_proxy is set if needed)
-   wget -P ${DATASET_DIR} https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz
-   tar -xzf ${DATASET_DIR}/flower_photos.tgz -C ${DATASET_DIR}
-
-   # Set the DATASET_DIR to the extracted images folder
-   DATASET_DIR=${DATASET_DIR}/flower_photos
-   ```
-
-   At this point, you should have a `flower_photos` folder with
-   subfolders for `daisy`, `dandelion`, `roses`, `sunflower`, and `tulips`.
-
 ## ③ Run the Intel Transfer Learning Tool
 
 With the Intel Transfer Learning Tool, you can train AI models with TensorFlow or
@@ -153,20 +131,23 @@ tlt list models --use-case image_classification
 
 **Train a Model**
 
-In this example, we'll use the ``tlt train`` command to use the TensorFlow
-ResNet50v1.5 model using the flowers dataset we already prepared and write the
-trained model to a folder specified with `--output-dir`.
-
+In this example, we'll use the `tlt train` command to retrain the TensorFlow
+ResNet50v1.5 model using a flowers dataset from the
+[TensorFlow Datasets catalog](https://www.tensorflow.org/datasets/catalog/tf_flowers).
+The `--dataset-dir` and `--output-dir` paths need to point to writable folders on your system.
 ```
-tlt train -f tensorflow --model-name resnet_v1_50 --dataset-dir ${DATASET_DIR} --output-dir /tmp/output
+# Use the follow environment variable setting to reduce the warnings and log output from TensorFlow
+export TF_CPP_MIN_LOG_LEVEL="2"
+
+tlt train -f tensorflow --model-name resnet_v1_50 --dataset-name tf_flowers --dataset-dir "/tmp/data-${USER}" --output-dir "/tmp/output-${USER}"
 ```
 ```
 Model name: resnet_v1_50
 Framework: tensorflow
+Dataset name: tf_flowers
 Training epochs: 1
-Dataset dir: /tmp/dataset/flower_photos
-Output directory: /tmp/output
-Found 3670 files belonging to 5 classes.
+Dataset dir: /tmp/data-user
+Output directory: /tmp/output-user
 ...
 Model: "sequential"
 _________________________________________________________________
@@ -179,9 +160,9 @@ Total params: 23,571,397
 Trainable params: 10,245
 Non-trainable params: 23,561,152
 _________________________________________________________________
-Checkpoint directory: /tmp/output/resnet_v1_50_checkpoints
+Checkpoint directory: /tmp/output-user/resnet_v1_50_checkpoints
 86/86 [==============================] - 24s 248ms/step - loss: 0.4600 - acc: 0.8438
-Saved model directory: /tmp/output/resnet_v1_50/1
+Saved model directory: /tmp/output-user/resnet_v1_50/1
 ```
 
 After training completes, the `tlt train` command evaluates the model. The loss and
@@ -217,22 +198,27 @@ from tlt.datasets import dataset_factory
 from tlt.models import model_factory
 from tlt.utils.types import FrameworkType, UseCaseType
 
-# Specify the directory where the TensorFlow flowers dataset has been downloaded and extracted
-# (https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz)
-dataset_dir = os.environ["DATASET_DIR"] if "DATASET_DIR" in os.environ else \
-    os.path.join(os.environ["HOME"], "dataset")
+username = os.getenv('USER', 'user')
+
+# Specify a writable directory for the dataset to be downloaded
+dataset_dir = '/tmp/data-{}'.format(username)
+if not os.path.exists(dataset_dir):
+    os.makedirs(dataset_dir)
 
-# Specify a directory for output
-output_dir = os.environ["OUTPUT_DIR"] if "OUTPUT_DIR" in os.environ else \
-    os.path.join(os.environ["HOME"], "output")
+# Specify a writeable directory for output (such as saved model files)
+output_dir = '/tmp/output-{}'.format(username)
+if not os.path.exists(output_dir):
+    os.makedirs(output_dir)
 
 # Get the model
 model = model_factory.get_model(model_name="resnet_v1_50", framework=FrameworkType.TENSORFLOW)
 
-# Load and preprocess a dataset
-dataset = dataset_factory.load_dataset(dataset_dir = os.path.join(dataset_dir, "flower_photos"),
-                                       use_case=UseCaseType.IMAGE_CLASSIFICATION, \
-                                       framework=FrameworkType.TENSORFLOW)
+# Download and preprocess the flowers dataset from the TensorFlow datasets catalog
+dataset = dataset_factory.get_dataset(dataset_dir=dataset_dir,
+                                      dataset_name='tf_flowers',
+                                      use_case=UseCaseType.IMAGE_CLASSIFICATION,
+                                      framework=FrameworkType.TENSORFLOW,
+                                      dataset_catalog='tf_datasets')
 dataset.preprocess(image_size=model.image_size, batch_size=32)
 dataset.shuffle_split(train_pct=.75, val_pct=.25)
 

README.md

@@ -1,4 +1,9 @@
+*Note: You may find it easier to read about Intel Transfer Learning tool, follow the Get
+Started guide, and browse the API material from our published documentation site
+https://intelai.github.io/transfer-learning.*
+
 <!-- SkipBadges -->
+
 # Intel® Transfer Learning Tool
 
 Transfer learning workflows use the knowledge learned by a pre-trained model on
@@ -55,15 +60,15 @@ figure:
 
 ## Get Started
 
-The [Get Started](GetStarted.md) guide walks you through the steps to check
-system requirements, install, and then run the tool with a couple of examples
-showing no-code CLI and low-code API approaches. After that, you can check out
+Check out the [Get Started Guide](GetStarted.md) which will walk you through the
+steps to check system requirements, install, and then run the tool with a couple of
+examples showing no-code CLI and low-code API approaches. After that, you can check out
 these additional CLI and API [Examples](examples/README.md).
 
 <!-- ExpandGetStarted-Start -->
-As described in the [Get Started](GetStarted.md) guide, once you have a Python
-3.9 environment set up, you do a basic install of the Intel Transfer Learning
-Tool using:
+As described in the [Get Started Guide](GetStarted.md), once you have a Python 
+environment set up, you do a basic install of the Intel Transfer Learning
+Tool. Here are some examples of commands you will find in the [Get Started Guide](GetStarted.md):
 
 ```
 pip install intel-transfer-learning-tool

docker/README.md

@@ -24,9 +24,6 @@ docker compose build
 OR
 ```bash
 docker pull intel/ai-tools:tlt-0.5.0
-docker pull intel/ai-tools:tlt-devel-0.5.0
-docker pull intel/ai-tools:tlt-dist-0.5.0
-docker pull intel/ai-tools:tlt-dist-devel-0.5.0
 ```
 
 ## Use Docker Image
@@ -56,11 +53,19 @@ OR
 helm repo add cowboysysop https://cowboysysop.github.io/charts/
 helm install <release name> cowboysysop/training-operator
 ```
+
+### 3. Build Distributed Container
+```bash
+cd docker
+docker compose build
+docker push <registry>:tlt-dist-latest
+```
+
 ### 3. Deploy TLT Distributed Job
 For more customization information, see the chart [README](./docker/chart/README.md)
 ```bash
 export NAMESPACE=kubeflow
-helm install --namespace ${NAMESPACE} --set ... tlt-distributed ./docker/chart
+helm install --namespace ${NAMESPACE} --set imageName=<registry> --set imageTag=tlt-dist-latest --set ... tlt-distributed ./docker/chart
 ```
 ### 4. View 
 To view your workflow progress

examples/cli/README.md

@@ -60,10 +60,10 @@ wget -P ${DATASET_DIR} https://storage.googleapis.com/download.tensorflow.org/ex
 tar -xzf ${DATASET_DIR}/flower_photos.tgz -C ${DATASET_DIR}
 
 # Set the DATASET_DIR to the extracted images folder
-DATASET_DIR=${DATASET_DIR}/flower_photos
+export DATASET_DIR=${DATASET_DIR}/flower_photos
 
 # Supress debug information from TensorFlow 2.12
-TF_CPP_MIN_LOG_LEVEL=2
+export TF_CPP_MIN_LOG_LEVEL=2
 ```
 
 After the dataset directory is ready, use the `tlt train` command to train one of the models from

examples/cli/text_classification.md

@@ -20,8 +20,8 @@ the label (`ham` or `spam`) and the second column is the text of the SMS message
 labels are replaced with numerical values before training.
 ```bash
 # Create dataset and output directories
-DATASET_DIR=/tmp/data
-OUTPUT_DIR=/tmp/output
+export DATASET_DIR=/tmp/data
+export OUTPUT_DIR=/tmp/output
 mkdir -p ${DATASET_DIR}
 mkdir -p ${OUTPUT_DIR}
 
@@ -71,8 +71,8 @@ and [glue/cola](https://www.tensorflow.org/datasets/catalog/glue#gluecola_defaul
 
 ```bash
 # Create dataset and output directories
-DATASET_DIR=/tmp/data
-OUTPUT_DIR=/tmp/output
+export DATASET_DIR=/tmp/data
+export OUTPUT_DIR=/tmp/output
 mkdir -p ${DATASET_DIR}
 mkdir -p ${OUTPUT_DIR}
 
@@ -114,8 +114,8 @@ one epoch using 2 nodes and 2 processes per node.
 
 ```bash
 # Create dataset and output directories
-DATASET_DIR=/tmp/data
-OUTPUT_DIR=/tmp/output
+export DATASET_DIR=/tmp/data
+export OUTPUT_DIR=/tmp/output
 mkdir -p ${DATASET_DIR}
 mkdir -p ${OUTPUT_DIR}
 

notebooks/requirements.txt

@@ -6,7 +6,8 @@ gin-config~=0.5.0
 intel-extension-for-pytorch==1.13.100
 intel-tensorflow==2.12.0
 ipython-genutils~=0.2.0
-ipython~=8.13.2
+ipython~=8.12.2; python_version<'3.9'
+ipython~=8.13.2; python_version>='3.9'
 ipywidgets~=8.0.6
 jmespath~=1.0.1
 matplotlib-inline~=0.1.6

notebooks/setup.md

@@ -2,44 +2,49 @@
 
 Use the instructions below to install the dependencies required to run the notebooks.
 
-System Requirements:
-1. Ubuntu 20.04
+Software Requirements:
+1. Linux* system (validated on Ubuntu* 20.04/22.04 LTS)
 2. Python3 (3.8, 3.9, or 3.10), Pip/Conda and Virtualenv
 3. git
 
 ## Set Up Notebook Environment
 
-1. Install Intel® Transfer Learning Tool using the Developer Installation option in the [Get Started](/GetStarted.md) Guide.
-   This is required for the Intel Transfer Learning Tool tutorial notebooks, E2E notebooks, and performance comparison. Follow the
-   instructions in the [Get Started Guide](/GetStarted.md). You can
-   skip this step if you are only running the native framework notebooks.
+1. Install Intel® Transfer Learning Tool using any of the installation options in the [Get Started Guide](/GetStarted.md).
+   This is required for the Intel Transfer Learning Tool tutorial notebooks, E2E notebooks, and performance comparison. 
+   You can skip this step if you are only running the native framework notebooks.
 
-2. Activate the virtualenv or conda environment used to install Intel Transfer Learning Tool,
+2. Clone the GitHub repo if you haven't done this in step 1
+
+   ```
+   git clone https://github.com/IntelAI/transfer-learning.git
+   cd transfer-learning 
+   ```
+
+3. Activate the virtualenv or conda environment used to install Intel Transfer Learning Tool,
    then from inside the activated environment, run these steps:
    ```
    pip install --upgrade pip
    pip install -r notebooks/requirements.txt
    ```
 
-3. Set environment variables for the path to the dataset folder and an output directory.
+4. Set environment variables for the path to the dataset folder and an output directory.
    The dataset and output directories can be empty. The notebook will download the dataset to
    the dataset directory, if it is empty. Subsequent runs will reuse the dataset.
    If the `DATASET_DIR` and `OUTPUT_DIR` variables are not defined, the notebooks will
    default to use `~/dataset` and `~/output`.
    ```
-   export DATASET_DIR=<directory to download the dataset>
-   export OUTPUT_DIR=<output directory for the saved model>
-
+   export DATASET_DIR=~/dataset
+   export OUTPUT_DIR=~/output
    mkdir -p $DATASET_DIR
    mkdir -p $OUTPUT_DIR
    ```
-4. Navigate to the notebook directory in your clone of the Transfer Learning repo, and then start the
+5. Navigate to the notebook directory in your clone of the Transfer Learning repo, and then start the
    [notebook server](https://jupyter.readthedocs.io/en/latest/running.html#starting-the-notebook-server):
    ```
    cd notebooks
    jupyter notebook --port 8888
    ```
-5. Copy and paste the URL from the terminal to your browser to view and run the notebooks.
+6. Copy and paste the URL from the terminal to your browser to view and run the notebooks.
 
 Once you have the environment and dependencies set up, see the list of available
-notebook examples.
+[notebooks](/notebooks/README.md).

notebooks/text_classification/tlt_api_tf_text_classification/TLT_TF_Text_Classification.ipynb

@@ -202,9 +202,9 @@
    "id": "ccac8980",
    "metadata": {},
    "source": [
-    "### Option B: Use the TFDS catalog\n",
+    "### Option B: Use the TensorFlow datasets catalog\n",
     "\n",
-    "Option B allows for using a dataset from the [TensorFlow datasets catalog](https://www.tensorflow.org/datasets/catalog/overview). The dataset factory currently supports the following TFDS text classification datasets: [imdb_reviews](https://www.tensorflow.org/datasets/catalog/imdb_reviews), [glue/sst2](https://www.tensorflow.org/datasets/catalog/imdb_reviews), [glue/cola](https://www.tensorflow.org/datasets/catalog/glue#gluecola_default_config), and [ag_news_subset](https://www.tensorflow.org/datasets/catalog/ag_news_subset)."
+    "Option B allows for using a dataset from the [TensorFlow datasets catalog](https://www.tensorflow.org/datasets/catalog/overview). The dataset factory currently supports the following TFDS text classification datasets: [imdb_reviews](https://www.tensorflow.org/datasets/catalog/imdb_reviews), [glue/sst2](https://www.tensorflow.org/datasets/catalog/glue#gluesst2), [glue/cola](https://www.tensorflow.org/datasets/catalog/glue#gluecola_default_config), and [ag_news_subset](https://www.tensorflow.org/datasets/catalog/ag_news_subset)."
    ]
   },
   {

tests/tools/cli/test_train_cli.py

@@ -238,7 +238,8 @@ def test_train_init_checkpoints(mock_load_dataset, mock_get_model, model_name, f
             model_mock.train.assert_called_once_with(data_mock, output_dir=output_dir, epochs=2,
                                                      initial_checkpoints=init_checkpoints, early_stopping=False,
                                                      lr_decay=False, ipex_optimize=False, distributed=False,
-                                                     hostfile=None, nnodes=1, nproc_per_node=1)
+                                                     hostfile=None, nnodes=1, nproc_per_node=1, use_horovod=False,
+                                                     hvd_start_timeout=30)
         data_mock.preprocess.assert_called_once_with(batch_size=32)
 
         # Verify that the train command exit code is successful
@@ -314,7 +315,8 @@ def test_train_features(mock_inspect, mock_load_dataset, mock_get_model, model_n
             model_mock.train.assert_called_once_with(data_mock, output_dir=output_dir, epochs=15,
                                                      initial_checkpoints=None, early_stopping=early_stopping,
                                                      lr_decay=lr_decay, ipex_optimize=False, distributed=False,
-                                                     hostfile=None, nnodes=1, nproc_per_node=1)
+                                                     hostfile=None, nnodes=1, nproc_per_node=1, use_horovod=False,
+                                                     hvd_start_timeout=30)
 
         # Verify that the train command exit code is successful
         assert result.exit_code == 0

tests/utils/test_file_utils.py

@@ -24,7 +24,7 @@
 import tempfile
 from unittest.mock import MagicMock
 
-from tlt.utils.file_utils import validate_model_name, download_file
+from tlt.utils.file_utils import download_file, get_model_name_from_path, validate_model_name
 
 
 @pytest.mark.common
@@ -70,3 +70,17 @@ def test_download():
     # Delete the temp output directory
     if os.path.exists(output_dir) and os.path.isdir(output_dir):
         shutil.rmtree(output_dir)
+
+
+@pytest.mark.common
+@pytest.mark.parametrize('model_dir,expected_model_name',
+                         [['/tmp/user/resnet_v2_50/12/', 'resnet_v2_50'],
+                          ['/tmp/user/resnet_v2_50/12', 'resnet_v2_50'],
+                          ['/localdisk/folder/google_bert_uncased_L-2_H-128_A-2/8/',
+                           'google_bert_uncased_L-2_H-128_A-2']])
+def test_get_model_name_from_path(model_dir, expected_model_name):
+    """
+    Tests the file utils method that returns the model name from a model directory path. Verifies that the model name
+    returned matches the expected model name.
+    """
+    assert expected_model_name == get_model_name_from_path(model_dir)