Fetch Firmware Release Data #25
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch Firmware Release Data | |
| on: | |
| schedule: | |
| - cron: '22 12 1 * *' | |
| - cron: '22 12 8 * *' | |
| - cron: '22 12 18 * *' | |
| - cron: '22 12 28 * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| fetch_data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/[email protected] | |
| - name: Setup Node.js | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '20' | |
| - name: Fetch and save releases data | |
| run: | | |
| mkdir -p assets/data | |
| node -e ' | |
| const fs = require("fs"); | |
| const https = require("https"); | |
| function cleanReleases(releases) { | |
| if (Array.isArray(releases)) { | |
| releases.forEach(release => { | |
| if (release && typeof release === "object") { | |
| delete release.body; | |
| delete release.author; | |
| delete release.mentions_count; | |
| delete release.tarball_url; | |
| delete release.zipball_url; | |
| } | |
| }); | |
| } | |
| return releases; | |
| } | |
| async function fetchPage(url, timeoutMs = 9000000) { | |
| return new Promise((resolve, reject) => { | |
| const req = https.get(url, { | |
| headers: { | |
| "User-Agent": "classicrocker883-firmware-selector" | |
| } | |
| }, (res) => { | |
| let data = ""; | |
| res.on("data", (chunk) => { data += chunk; }); | |
| res.on("end", () => { | |
| clearTimeout(timeoutId); | |
| if (res.statusCode >= 200 && res.statusCode < 300) { | |
| try { | |
| resolve(JSON.parse(data)); | |
| } catch (e) { | |
| reject(new Error(`Failed to parse JSON from ${url}: ${e.message}`)); | |
| } | |
| } else { | |
| reject(new Error(`Failed to fetch ${url}, Status: ${res.statusCode}, Message: ${data}`)); | |
| } | |
| }); | |
| }).on("error", (err) => { | |
| clearTimeout(timeoutId); | |
| reject(err); | |
| }); | |
| const timeoutId = setTimeout(() => { | |
| req.destroy(new Error(`Request timed out after ${timeoutMs}ms`)); | |
| }, timeoutMs); | |
| }); | |
| } | |
| async function fetchAllReleases(url, releases = [], sinceDate = null) { | |
| let currentPageUrl = `${url}`; | |
| console.log(`Fetching from: ${currentPageUrl}`); | |
| const data = await fetchPage(currentPageUrl); | |
| if (data.length === 0) { | |
| return releases; | |
| } | |
| const newReleases = []; | |
| let reachedOlderReleases = false; | |
| for (const release of data) { | |
| if (sinceDate && new Date(release.published_at) < sinceDate) { | |
| reachedOlderReleases = true; | |
| break; | |
| } | |
| newReleases.push(release); | |
| } | |
| const allReleases = releases.concat(newReleases); | |
| if (reachedOlderReleases || data.length < 100) { | |
| return allReleases; | |
| } else { | |
| return fetchAllReleases(url, allReleases, sinceDate); | |
| } | |
| } | |
| async function main() { | |
| const repoApiUrl = "https://api.github.com/repos/classicrocker883/MRiscoCProUI/releases"; | |
| const latestReleaseApiUrl = "https://api.github.com/repos/classicrocker883/MRiscoCProUI/releases/latest"; | |
| const filePath = "assets/data/releases.json"; | |
| let existingReleases = []; | |
| const sixMonthsAgo = new Date(); | |
| sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6); | |
| console.log(`Limiting fetch to releases published after: ${sixMonthsAgo.toISOString()}`); | |
| if (fs.existsSync(filePath)) { | |
| try { | |
| const fileContent = fs.readFileSync(filePath, "utf8"); | |
| if (fileContent) { | |
| existingReleases = JSON.parse(fileContent); | |
| console.log(`Loaded ${existingReleases.length} existing releases from ${filePath}.`); | |
| } | |
| } catch (parseError) { | |
| console.warn(`Could not parse existing releases.json (${parseError.message}), starting fresh.`); | |
| existingReleases = []; | |
| } | |
| } else { | |
| console.log("No existing releases.json found. Fetching all releases published in the last 6 months."); | |
| } | |
| try { | |
| let absoluteLatestRelease = null; | |
| try { | |
| absoluteLatestRelease = await fetchPage(latestReleaseApiUrl); | |
| console.log(`Fetched absolute latest release: ${absoluteLatestRelease.tag_name}`); | |
| } catch (latestError) { | |
| console.warn(`Could not fetch latest release from ${latestReleaseApiUrl}: ${latestError.message}. Proceeding without it.`); | |
| } | |
| const allPaginatedReleases = await fetchAllReleases(repoApiUrl, [], sixMonthsAgo); | |
| console.log(`Fetched ${allPaginatedReleases.length} paginated release(s).`); | |
| const allReleasesMap = new Map(); | |
| existingReleases.forEach(release => allReleasesMap.set(release.id, release)); | |
| allPaginatedReleases.forEach(release => allReleasesMap.set(release.id, release)); | |
| if (absoluteLatestRelease) { | |
| allReleasesMap.set(absoluteLatestRelease.id, absoluteLatestRelease); | |
| } | |
| let combinedReleases = Array.from(allReleasesMap.values()); | |
| console.log(`Total unique releases after merging: ${combinedReleases.length}.`); | |
| let finalSortedReleases; | |
| if (absoluteLatestRelease) { | |
| const releasesToSort = combinedReleases.filter(r => r.id !== absoluteLatestRelease.id); | |
| releasesToSort.sort((a, b) => new Date(b.published_at) - new Date(a.published_at)); | |
| finalSortedReleases = [absoluteLatestRelease, ...releasesToSort]; | |
| } else { | |
| combinedReleases.sort((a, b) => new Date(b.published_at) - new Date(a.published_at)); | |
| finalSortedReleases = combinedReleases; | |
| } | |
| const cleanedReleases = cleanReleases(finalSortedReleases); | |
| let hasChanges = true; | |
| if (fs.existsSync(filePath)) { | |
| const currentFileContent = fs.readFileSync(filePath, "utf8"); | |
| if (currentFileContent === JSON.stringify(cleanedReleases, null, 2)) { | |
| hasChanges = false; | |
| console.log("No changes detected. The file is already up to date."); | |
| } | |
| } | |
| if (hasChanges) { | |
| if (!fs.existsSync("assets/data")) { | |
| fs.mkdirSync("assets/data", { recursive: true }); | |
| } | |
| fs.writeFileSync(filePath, JSON.stringify(cleanedReleases, null, 2)); | |
| console.log("Successfully updated and saved releases.json."); | |
| } else { | |
| console.log("No new data to commit."); | |
| } | |
| } catch (error) { | |
| console.error("Failed to fetch or save releases:", error.message); | |
| process.exit(1); | |
| } | |
| } | |
| main(); | |
| ' | |
| - name: Commit and push if changes | |
| run: | | |
| git config user.name "Andrew" | |
| git config user.email "[email protected]" | |
| git add assets/data/releases.json | |
| if git diff --cached --exit-code assets/data/releases.json; then | |
| echo "No changes to commit." | |
| else | |
| git commit -m "automated: update firmware release data" | |
| git push | |
| echo "Changes committed and pushed." | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |