Skip to content

Commit 926f8ea

Browse files
oluptonolupton
authored
Gemma: remove container build and CI (#1429)
Gemma models are still available via MaxText, this simply removes the container and tests of the https://github.com/google-deepmind/gemma repository, which were not actively maintained.
1 parent 589cea1 commit 926f8ea

File tree

8 files changed

+1
-1670
lines changed

8 files changed

+1
-1670
lines changed

‎.github/container/manifest.yaml‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -81,46 +81,16 @@ maxtext:
8181
tracking_ref: main
8282
latest_verified_commit: 78daad198544def8274dbd656d122fbe6a0e1129
8383
mode: git-clone
84-
haliax:
85-
url: https://github.com/stanford-crfm/haliax.git
86-
tracking_ref: main
87-
latest_verified_commit: 2a696a0c971901ff93afdaa965959d8e3b982ba9
88-
mode: git-clone
8984
mlperf-logging:
9085
url: https://github.com/mlcommons/logging.git
9186
tracking_ref: master
9287
latest_verified_commit: 99ba37ac267c870d7c6c17e1837aa9180a37cdc1
9388
mode: pip-vcs
94-
gemma:
95-
url: https://github.com/google-deepmind/gemma.git
96-
tracking_ref: main
97-
latest_verified_commit: 036083ab16843e09369a0138630687dba96d4d23
98-
mode: git-clone
9989
equinox:
10090
url: https://github.com/patrick-kidger/equinox.git
10191
tracking_ref: main
10292
latest_verified_commit: 1e601672d38d2c4d483535070a3572d8e8508a20
10393
mode: git-clone
104-
big-vision:
105-
url: https://github.com/google-research/big_vision.git
106-
tracking_ref: main
107-
latest_verified_commit: 70b22bb60c46023e50fd0f5d42e0f48f8dfe2fdb
108-
mode: git-clone
109-
common-loop-utils:
110-
url: https://github.com/google/CommonLoopUtils.git
111-
tracking_ref: main
112-
latest_verified_commit: b172f5a21fbd6236ea9408ed94d0fc0b11bffe33
113-
mode: git-clone
114-
flaxformer:
115-
url: https://github.com/google/flaxformer.git
116-
tracking_ref: main
117-
latest_verified_commit: 399ea3a85e9807ada653fd0de1a9de627eb0acde
118-
mode: git-clone
119-
panopticapi:
120-
url: https://github.com/akolesnikoff/panopticapi.git
121-
tracking_ref: mute
122-
latest_verified_commit: a698a12deb21e4cf0f99ef0581b2c30c466bf355
123-
mode: git-clone
12494
orbax-checkpoint:
12595
url: https://github.com/google/orbax.git
12696
tracking_ref: main

‎.github/workflows/_ci.yaml‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -122,27 +122,6 @@ jobs:
122122
BASE_LIBRARY: t5x
123123
secrets: inherit
124124

125-
build-gemma:
126-
needs: build-jax
127-
uses: ./.github/workflows/_build.yaml
128-
if: inputs.ARCHITECTURE == 'amd64' # build only amd64
129-
with:
130-
ARCHITECTURE: ${{ inputs.ARCHITECTURE }}
131-
ARTIFACT_NAME: artifact-gemma-build
132-
BADGE_FILENAME: badge-gemma-build
133-
BUILD_DATE: ${{ inputs.BUILD_DATE }}
134-
BASE_IMAGE: ${{ needs.build-jax.outputs.DOCKER_TAG_MEALKIT }}
135-
CONTAINER_NAME: gemma
136-
DOCKERFILE: rosetta/Dockerfile.gemma
137-
DOCKER_CONTEXT: .
138-
EXTRA_BUILD_ARGS: |
139-
URLREF_GEMMA=${{ fromJson(inputs.SOURCE_URLREFS).GEMMA }}
140-
URLREF_BIG_VISION=${{ fromJson(inputs.SOURCE_URLREFS).BIG_VISION }}
141-
URLREF_COMMON_LOOP_UTILS=${{ fromJson(inputs.SOURCE_URLREFS).COMMON_LOOP_UTILS }}
142-
URLREF_FLAXFORMER=${{ fromJson(inputs.SOURCE_URLREFS).FLAXFORMER }}
143-
URLREF_PANOPTICAPI=${{ fromJson(inputs.SOURCE_URLREFS).PANOPTICAPI }}
144-
secrets: inherit
145-
146125
build-axlearn:
147126
needs: build-jax
148127
uses: ./.github/workflows/_build.yaml
@@ -167,7 +146,6 @@ jobs:
167146
- build-maxtext
168147
- build-upstream-t5x
169148
- build-rosetta-t5x
170-
- build-gemma
171149
- build-axlearn
172150
outputs:
173151
TAGS: ${{ steps.collect-tags.outputs.TAGS }}
@@ -183,14 +161,12 @@ jobs:
183161
{"flavor": "maxtext", "stage": "final", "priority": 900, "tag": "${{ needs.build-maxtext.outputs.DOCKER_TAG_FINAL }}"},\
184162
{"flavor": "upstream-t5x", "stage": "final", "priority": 900, "tag": "${{ needs.build-upstream-t5x.outputs.DOCKER_TAG_FINAL }}"},\
185163
{"flavor": "t5x", "stage": "final", "priority": 900, "tag": "${{ needs.build-rosetta-t5x.outputs.DOCKER_TAG_FINAL }}"},\
186-
{"flavor": "gemma", "stage": "final", "priority": 900, "tag": "${{ needs.build-gemma.outputs.DOCKER_TAG_FINAL }}"},\
187164
{"flavor": "axlearn", "stage": "final", "priority": 900, "tag": "${{ needs.build-axlearn.outputs.DOCKER_TAG_FINAL }}"},\
188165
{"flavor": "jax", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-jax.outputs.DOCKER_TAG_MEALKIT }}"},\
189166
{"flavor": "equinox", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-equinox.outputs.DOCKER_TAG_MEALKIT }}"},\
190167
{"flavor": "maxtext", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-maxtext.outputs.DOCKER_TAG_MEALKIT }}"},\
191168
{"flavor": "upstream-t5x", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-upstream-t5x.outputs.DOCKER_TAG_MEALKIT }}"},\
192169
{"flavor": "t5x", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-rosetta-t5x.outputs.DOCKER_TAG_MEALKIT }}"},\
193-
{"flavor": "gemma", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-gemma.outputs.DOCKER_TAG_MEALKIT }}"},\
194170
{"flavor": "axlearn", "stage": "mealkit", "priority": 500, "tag": "${{ needs.build-axlearn.outputs.DOCKER_TAG_MEALKIT }}"},\
195171
196172
{}\
@@ -532,30 +508,6 @@ jobs:
532508
T5X_IMAGE: ${{ needs.build-rosetta-t5x.outputs.DOCKER_TAG_FINAL }}
533509
secrets: inherit
534510

535-
test-gemma:
536-
needs: build-gemma
537-
uses: ./.github/workflows/_test_unit.yaml
538-
if: inputs.ARCHITECTURE == 'amd64'
539-
with:
540-
TEST_NAME: gemma
541-
EXECUTE: |
542-
docker run --shm-size=1g --gpus all ${{ needs.build-gemma.outputs.DOCKER_TAG_FINAL }} \
543-
bash -ec \
544-
"cd /opt/gemma && pip install -e .[dev] && pytest ." | tee test-gemma.log
545-
STATISTICS_SCRIPT: |
546-
summary_line=$(tail -n1 test-gemma.log)
547-
errors=$(echo $summary_line | grep -oE '[0-9]+ error' | awk '{print $1} END { if (!NR) print 0}')
548-
failed_tests=$(echo $summary_line | grep -oE '[0-9]+ failed' | awk '{print $1} END { if (!NR) print 0}')
549-
passed_tests=$(echo $summary_line | grep -oE '[0-9]+ passed' | awk '{print $1} END { if (!NR) print 0}')
550-
total_tests=$((failed_tests + passed_tests))
551-
echo "TOTAL_TESTS=${total_tests}" >> $GITHUB_OUTPUT
552-
echo "ERRORS=${errors}" >> $GITHUB_OUTPUT
553-
echo "PASSED_TESTS=${passed_tests}" >> $GITHUB_OUTPUT
554-
echo "FAILED_TESTS=${failed_tests}" >> $GITHUB_OUTPUT
555-
ARTIFACTS: |
556-
test-gemma.log
557-
secrets: inherit
558-
559511
test-maxtext:
560512
needs: build-maxtext
561513
if: inputs.ARCHITECTURE == 'amd64' # no arm64 gpu runners

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ on:
3737
type: string
3838
description: |
3939
A comma-separated PACKAGE=URL#REF list to override sources used by build.
40-
PACKAGE∊{JAX,XLA,Flax,transformer-engine,T5X,maxtext,haliax,mujuco,mujuco-mpc,gemma,big-vision,common-loop-utils,flaxformer,panopticapi} (case-insensitive)
40+
PACKAGE∊{JAX,XLA,Flax,transformer-engine,airio,axlearn,equinox,T5X,maxtext} (case-insensitive)
4141
default: ''
4242
required: false
4343

@@ -345,7 +345,6 @@ jobs:
345345
maxtext
346346
upstream-t5x
347347
t5x
348-
gemma
349348
)
350349
declare -a STAGES=(
351350
mealkit

‎README.md‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ We support and test the following JAX frameworks and model architectures. More d
1313
| [maxtext](./rosetta/rosetta/projects/maxtext)| GPT, LLaMA, Gemma, Mistral, Mixtral | pre-training | `ghcr.io/nvidia/jax:maxtext` |
1414
| [t5x](./rosetta/rosetta/projects/t5x) | T5, ViT | pre-training, fine-tuning | `ghcr.io/nvidia/jax:t5x` |
1515
| [t5x](./rosetta/rosetta/projects/imagen) | Imagen | pre-training | `ghcr.io/nvidia/t5x:imagen-2023-10-02.v3` |
16-
| [big vision](./rosetta/rosetta/projects/paligemma) | PaliGemma | fine-tuning, evaluation | `ghcr.io/nvidia/jax:gemma` |
1716
| [axlearn](./docs/frameworks/axlearn/README.md) | Fuji | pre-training | `ghcr.io/nvidia/jax:axlearn` |
1817

1918
# Build Pipeline Status
@@ -180,27 +179,6 @@ We support and test the following JAX frameworks and model architectures. More d
180179
</a>
181180
</td>
182181
</tr>
183-
<tr>
184-
<td>
185-
<a href="https://github.com/NVIDIA/JAX-Toolbox/blob/main/rosetta/Dockerfile.gemma">
186-
<img style="height:1em;" src="https://img.shields.io/static/v1?label=&color=gray&logo=docker&message=Gemma%3D%7Bcore%2CGemma%2CPaliGemma%7D">
187-
</a>
188-
</td>
189-
<td>
190-
<code>ghcr.io/nvidia/jax:gemma</code>
191-
</td>
192-
<td>
193-
<a href="https://gist.github.com/nvjax/913c2af68649fe568e9711c2dabb23ae/#file-final-gemma-md">
194-
<img style="height:1em;" src="https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2Fnvjax%2F913c2af68649fe568e9711c2dabb23ae%2Fraw%2Fbadge-gemma-build-amd64.json&logo=docker&label=amd64">
195-
</a>
196-
<!-- <a href="https://gist.github.com/nvjax/913c2af68649fe568e9711c2dabb23ae/#file-final-gemma-md"><img style="height:1em;" src="https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2Fnvjax%2F913c2af68649fe568e9711c2dabb23ae%2Fraw%2Fbadge-gemma-build-arm64.json&logo=docker&label=arm64"></a> -->
197-
</td>
198-
<td>
199-
<a href="https://gist.github.com/nvjax/913c2af68649fe568e9711c2dabb23ae#file-badge-gemma-unit-test-a100-json">
200-
<img style="height:1em;" src="https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2Fnvjax%2F913c2af68649fe568e9711c2dabb23ae%2Fraw%2Fbadge-gemma-unit-test-A100.json&logo=nvidia&label=A100">
201-
</a>
202-
</td>
203-
</tr>
204182
<tr>
205183
<td>
206184
<a href="https://github.com/NVIDIA/JAX-Toolbox/blob/main/.github/container/Dockerfile.axlearn">

‎rosetta/Dockerfile.gemma‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

0 commit comments

Comments
 (0)