Content staging

.github/workflows/deploy-content-staging.yml

@@ -1,87 +1,135 @@
-name: Content Staging CI
+name: Automated Content Update
 
 on:
-  push:
-    branches: [ content-staging ]
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+  schedule:
+    - cron: '0 */12 * * *'
 
 jobs:
-  buildAndDeploy:
+  update-content:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      - name: Add SSH private keys for submodule repositories
-        uses: webfactory/ssh-agent@v0.7.0
-        with:
-          ssh-private-key: |
-            ${{ secrets.OATUTOR_CONTENT_DEPLOY_KEY }}
-
-      - run: git submodule update --init --recursive
-
-      - name: Save original git author and git email
-        run: |
-          echo "git_email=$(git log --format='%ae' HEAD^!)" >> $GITHUB_ENV
-          echo "git_name=$(git log --format='%an' HEAD^!)" >> $GITHUB_ENV
-
-      - name: Get build time
-        run: echo "build_timestamp=$(date +%s%3N)" >> $GITHUB_ENV
-
-      - uses: actions/setup-node@v2.4.1
-
-      - name: Change homepage in package.json
-        run: sed -i 's/place-holder/OATutor-Content-Staging/g' package.json
-
-#      - name: Do not log data
-#        run: sed -i 's/DO_LOG_DATA = true/DO_LOG_DATA = false/g' src/config/config.js
-
-      - name: Cache node modules
-        uses: actions/cache@v2
-        env:
-            cache-name: cache-node-modules
-        with:
-            # npm cache files are stored in `~/.npm` on Linux/macOS
-            path: ~/.npm
-            key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-            restore-keys: |
-                ${{ runner.os }}-build-${{ env.cache-name }}-
-                ${{ runner.os }}-build-
-                ${{ runner.os }}-
-
-      - name: Run CI install
-        run: npm ci
-
-      - name: Run build
-        run: npm run build
-        env:
-          REACT_APP_FIREBASE_CONFIG: ${{ secrets.STAGING_FIREBASE_CONFIG }}
-          CI: false
-          REACT_APP_BUILD_TYPE: "content-staging"
-          REACT_APP_MIDDLEWARE_URL: ${{ secrets.STAGING_MIDDLEWARE_URL }}
-          REACT_APP_COMMIT_HASH: ${{ github.sha }}
-          REACT_APP_BUILD_TIMESTAMP: ${{ env.build_timestamp }}
-
-      - uses: actions/checkout@v2
-        with:
-          repository: 'CAHLR/OATutor-Content-Staging'
-          ssh-key: ${{ secrets.CONTENT_STAGING_SSH_PRIVATE_KEY }}
-          ref: 'main'
-          path: 'content-staging-build'
-
-      - name: Push
-        run: |
-          cd content-staging-build
-          git rm -rf .
-          git clean -fxd
-          rsync -av ../build/ .
-          git add .
-          git config --local user.email "${{ env.git_email }}"
-          git config --local user.name "${{ env.git_name }}"
-          git diff --cached --quiet --exit-code && echo "no changes to platform, exiting early" && exit 0
-          git commit -m "deploy commit: CAHLR/OATutor@$GITHUB_SHA"
-          git push
+    - name: Checkout OATutor repository
+      uses: actions/checkout@v2
+      with:
+        ref: 'content-staging'
+        path: 'content-staging-build'
+        persist-credentials: true
+
+    - name: Clone OATutor-Tooling repository
+      run: git clone https://github.com/CAHLR/OATutor-Tooling.git /home/runner/work/OATutor-Tooling
+
+    - name: Install distutils and build tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3-distutils build-essential meson ninja-build python3-dev
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11'
+
+    - name: Upgrade pip and install Python dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --prefer-binary -r /home/runner/work/OATutor-Tooling/content_script/requirements.txt
+
+    - name: Create credentials JSON file
+      run: echo "${{ secrets.OATUTOR_JSON_KEY }}" | base64 --decode > /home/runner/work/oatutor-askoski-705644bfdf34.json
+
+    - name: Set environment variable for spreadsheet key
+      run: echo "URL_SPREADSHEET_KEY=${{ secrets.URL_SPREADSHEET_KEY }}" >> $GITHUB_ENV
+
+    - name: Remove existing content
+      run: rm -rf content-staging-build/src/content-sources/oatutor/*
+
+    - name: Create Content Directory
+      run: mkdir -p content-staging-build/src/content-sources/oatutor/Content
+
+    - name: Run content update script
+      run: |
+        cd content-staging-build/src/content-sources/oatutor/Content
+        python3 /home/runner/work/OATutor-Tooling/content_script/final.py online full
+
+    - name: Move and prepare files
+      run: |
+        cd content-staging-build/src/content-sources/oatutor
+        mv "OpenStax Content" "content-pool"
+        mkdir -p bkt-params
+        mv bktParams.json bkt-params/defaultBKTParams.json
+        cp bkt-params/defaultBKTParams.json bkt-params/experimentalBKTParams.json
+
+    - name: Run Node.js preprocessing script
+      run: |
+        cd content-staging-build/src/tools
+        node preprocessProblemPool.js
+
+    - name: Configure Git
+      run: |
+        cd content-staging-build
+        git config user.email "generic@example.com"
+        git config user.name "Generic User"
+
+    - name: Commit and push changes
+      run: |
+        cd content-staging-build
+        git add .
+        if git diff-index --quiet HEAD; then
+          echo "No changes to commit"
+        else
+          git commit -m "Automated content update"
+          git push origin content-staging
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.CONTENT_STAGING_PAT }}
+
+    - name: Change homepage in package.json
+      run: |
+        cd content-staging-build
+        sed -i 's/place-holder/OATutor-Content-Staging/g' package.json
+
+    - name: Run CI install
+      run: |
+        cd content-staging-build
+        npm ci
+
+    - name: Run build
+      run: |
+        cd content-staging-build
+        npm run build
+      env:
+        REACT_APP_FIREBASE_CONFIG: ${{ secrets.STAGING_FIREBASE_CONFIG }}
+        CI: false
+        REACT_APP_BUILD_TYPE: "content-staging"
+        REACT_APP_MIDDLEWARE_URL: ${{ secrets.STAGING_MIDDLEWARE_URL }}
+        REACT_APP_COMMIT_HASH: ${{ github.sha }}
+        REACT_APP_BUILD_TIMESTAMP: ${{ env.build_timestamp }}
+
+    - name: Checkout Content Staging
+      uses: actions/checkout@v3
+      with:
+        repository: 'CAHLR/OATutor-Content-Staging'
+        ref: 'main'
+        token: ${{ secrets.CONTENT_STAGING_PAT }}
+        path: 'content-staging-deploy'
+
+    - name: Save original git author and git email
+      run: |
+        echo "git_email=$(git log --format='%ae' HEAD^!)" >> $GITHUB_ENV
+        echo "git_name=$(git log --format='%an' HEAD^!)" >> $GITHUB_ENV
+
+    - name: Push changes
+      run: |
+        cd content-staging-deploy
+        git rm -rf .
+        git clean -fxd
+        rsync -av ../content-staging-build/build/ .
+        git add .
+        git config --global user.email "${{ env.git_email || 'oatutor@example.com' }}"
+        git config --global user.name "${{ env.git_name || 'OATutor' }}"
+        git diff --cached --quiet --exit-code && echo "no changes to platform, exiting early" && exit 0
+        git commit -m "deploy commit: CAHLR/OATutor@$GITHUB_SHA"
+        git push
+      env:
+        GITHUB_TOKEN: ${{ secrets.CONTENT_STAGING_PAT }}

.gitignore

@@ -28,6 +28,7 @@ credentials-secret.js
 service-account-credentials.json
 sheets-service-account.json
 oatutor-firebase-adminsdk.json
+oatutor-askoski-705644bfdf34.json
 
 # Misc.
 **/.vscode

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/content-sources/oatutor"]
-	path = src/content-sources/oatutor
-	url = git@github.com:CAHLR/OATutor-Content.git

logs/updateContent-2023-03-20_05-41-23.log

@@ -0,0 +1,8 @@
+[Error: ENOENT: no such file or directory, stat '/research/home/oatutor/dev/OATutor-Content-Staging/public/static/images/figures'] {
+  errno: -2,
+  code: 'ENOENT',
+  syscall: 'stat',
+  path: '/research/home/oatutor/dev/OATutor-Content-Staging/public/static/images/figures'
+}
+error removing existing figures
+writing to pool file...