diff --git a/README.md b/README.md
index 83ad86d1e7..a84153a092 100644
--- a/README.md
+++ b/README.md
@@ -178,6 +178,788 @@ Snap Users](./Music_Blocks_for_Snap_Users.md).
Looking for a block? Find it in the
[Palette Tables](./guide/README.md#6-appendix).
+## LEGO Bricks Widget
+
+The LEGO Bricks Widget is an innovative musical tool that combines visual image scanning with musical phrase creation. It allows users to upload images or use a webcam to scan visual content and convert color patterns into musical sequences. This widget represents a breakthrough in accessible music education, enabling users to create music through visual interaction.
+
+### Core Architecture and Data Structures
+
+The widget is built around a sophisticated matrix data structure that dynamically adapts to the pitch blocks provided by Music Blocks:
+
+```javascript
+this.matrixData = {
+ rows: [
+ { type: 'pitch', label: 'High C (Do)', icon: 'HighC.png', color: 'pitch-row', note: 'C5' },
+ { type: 'pitch', label: 'B (Ti)', icon: 'B.png', color: 'pitch-row', note: 'B4' },
+ // ... additional pitch rows based on user's pitch blocks
+ { type: 'control', label: 'Zoom Controls', icon: 'zoom.svg', color: 'control-row' }
+ ],
+ columns: 8,
+ selectedCells: new Set()
+};
+```
+
+The widget maintains several key properties for managing state:
+- **Widget Window Management**: Handles the visual interface and user interactions
+- **Audio Synthesis**: Integrated piano synthesizer for real-time note playback
+- **Image Processing**: Canvas-based color detection and analysis
+- **Animation Control**: Vertical scanning line animation system
+- **Data Storage**: Arrays for storing detected color patterns and musical notes
+
+### Widget Initialization and Setup
+
+The initialization process involves multiple stages of setup to create a fully functional musical interface:
+
+```javascript
+this.init = function(activity) {
+ this.activity = activity;
+ this.running = true;
+
+ // Initialize audio synthesizer with piano samples
+ this._initAudio();
+
+ // Create the main widget window
+ const widgetWindow = window.widgetWindows.windowFor(this, "LEGO BRICKS");
+ this.widgetWindow = widgetWindow;
+
+ // Add control buttons for various functions
+ this.playButton = widgetWindow.addButton("play-button.svg", ICONSIZE, _("Play"));
+ this.saveButton = widgetWindow.addButton("save-button.svg", ICONSIZE, _("Save"));
+ this.uploadButton = widgetWindow.addButton("upload-button.svg", ICONSIZE, _("Upload Image"));
+ this.webcamButton = widgetWindow.addButton("webcam-button.svg", ICONSIZE, _("Webcam"));
+
+ // Generate dynamic rows based on pitch blocks from Music Blocks
+ this._generateRowsFromPitchBlocks();
+
+ // Initialize the user interface components
+ this._initializeRowHeaders();
+ this.createMainContainer();
+}
+```
+
+### Dynamic Row Generation System
+
+The widget intelligently generates its musical rows based on the pitch blocks connected to it in Music Blocks:
+
+```javascript
+this._generateRowsFromPitchBlocks = function() {
+ this.matrixData.rows = [];
+
+ // Process each pitch block received from Music Blocks
+ for (let i = 0; i < this.rowLabels.length; i++) {
+ const pitchName = this.rowLabels[i];
+ const octave = this.rowArgs[i];
+
+ // Only process valid pitch blocks (skip drum blocks)
+ if (octave !== -1) {
+ const noteString = pitchName + octave;
+ const displayName = this._getDisplayName(pitchName, octave);
+
+ this.matrixData.rows.push({
+ type: 'pitch',
+ label: displayName,
+ icon: 'pitch.svg',
+ color: 'pitch-row',
+ note: noteString
+ });
+ }
+ }
+
+ // Add control row for zoom and spacing controls
+ this.matrixData.rows.push({
+ type: 'control',
+ label: 'Zoom Controls',
+ icon: 'zoom.svg',
+ color: 'control-row'
+ });
+};
+```
+
+### Advanced Color Detection System
+
+The color detection system is the heart of the widget, using sophisticated algorithms to analyze images:
+
+```javascript
+this._sampleAndDetectColor = function(line, now) {
+ // Create temporary canvas for pixel-level analysis
+ const tempCanvas = document.createElement('canvas');
+ const ctx = tempCanvas.getContext('2d');
+
+ // Set canvas dimensions to match media element
+ tempCanvas.width = mediaElement.naturalWidth || mediaElement.videoWidth || mediaRect.width;
+ tempCanvas.height = mediaElement.naturalHeight || mediaElement.videoHeight || mediaRect.height;
+
+ // Draw the current frame/image to canvas for analysis
+ ctx.drawImage(mediaElement, 0, 0);
+
+ // Sample multiple points along the vertical scanning line
+ const samplePoints = 32;
+ const colorCounts = {};
+ let totalSamples = 0;
+
+ for (let i = 0; i < samplePoints; i++) {
+ const y = canvasY1 + (i * (canvasY2 - canvasY1)) / samplePoints;
+
+ // Extract RGBA pixel data
+ const pixelData = ctx.getImageData(canvasX, y, 1, 1).data;
+ const [r, g, b, a] = pixelData;
+
+ // Skip transparent pixels
+ if (a < 128) continue;
+
+ // Convert RGB to color family
+ const colorFamily = this._getColorFamily(r, g, b);
+ colorCounts[colorFamily.name] = (colorCounts[colorFamily.name] || 0) + 1;
+ totalSamples++;
+ }
+
+ // Determine dominant color (must be at least 25% of samples)
+ let dominantColor = null;
+ let maxCount = 0;
+ const minThreshold = Math.max(1, Math.floor(totalSamples * 0.25));
+
+ for (const [colorName, count] of Object.entries(colorCounts)) {
+ if (count > maxCount && count >= minThreshold) {
+ maxCount = count;
+ dominantColor = this._getColorFamilyByName(colorName);
+ }
+ }
+
+ // Record significant color changes
+ if (dominantColor && (!line.currentColor || !this._colorsAreSimilar(line.currentColor, dominantColor))) {
+ this._addColorSegment(line.rowIndex, dominantColor, now - line.lastColorTime);
+ line.currentColor = dominantColor;
+ line.lastColorTime = now;
+ }
+};
+```
+
+### HSL Color Space Conversion and Analysis
+
+The widget uses HSL (Hue, Saturation, Lightness) color space for more accurate color classification:
+
+```javascript
+this._getColorFamily = function(r, g, b) {
+ // Convert RGB values to HSL for better color analysis
+ const hsl = this._rgbToHsl(r, g, b);
+ const [hue, saturation, lightness] = hsl;
+
+ // Handle grayscale colors first (low saturation)
+ if (saturation < 15) {
+ if (lightness > 85) return { name: 'white', hue: 0 };
+ if (lightness < 15) return { name: 'black', hue: 0 };
+ return { name: 'gray', hue: 0 };
+ }
+
+ // Classify saturated colors by hue ranges
+ if (hue >= 345 || hue < 15) return { name: 'red', hue: 0 };
+ if (hue >= 15 && hue < 45) return { name: 'orange', hue: 30 };
+ if (hue >= 45 && hue < 75) return { name: 'yellow', hue: 60 };
+ if (hue >= 75 && hue < 165) return { name: 'green', hue: 120 };
+ if (hue >= 165 && hue < 195) return { name: 'cyan', hue: 180 };
+ if (hue >= 195 && hue < 255) return { name: 'blue', hue: 240 };
+ if (hue >= 255 && hue < 285) return { name: 'purple', hue: 270 };
+ if (hue >= 285 && hue < 315) return { name: 'magenta', hue: 300 };
+ if (hue >= 315 && hue < 345) return { name: 'pink', hue: 330 };
+
+ return { name: 'unknown', hue: hue };
+};
+
+this._rgbToHsl = function(r, g, b) {
+ r /= 255; g /= 255; b /= 255;
+
+ const max = Math.max(r, g, b), min = Math.min(r, g, b);
+ let h = 0, s = 0, l = (max + min) / 2;
+
+ if (max !== min) {
+ const d = max - min;
+ s = l > 0.5 ? d / (2 - max - min) : d / (max + min);
+
+ switch (max) {
+ case r: h = (g - b) / d + (g < b ? 6 : 0); break;
+ case g: h = (b - r) / d + 2; break;
+ case b: h = (r - g) / d + 4; break;
+ }
+ h *= 60;
+ }
+
+ return [Math.round(h), Math.round(s * 100), Math.round(l * 100)];
+};
+```
+
+### Temporal Analysis and Musical Timing
+
+The widget converts time-based color segments into musical note durations using sophisticated filtering:
+
+```javascript
+this._collectNotesToPlay = function() {
+ this._notesToPlay = [];
+
+ // Analyze all color segment boundaries across rows
+ const columnBoundaries = this._analyzeColumnBoundaries();
+
+ // Filter out segments shorter than 350ms to eliminate noise
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ // Convert each time segment into a musical note
+ for (let colIndex = 0; colIndex < filteredBoundaries.length - 1; colIndex++) {
+ const startTime = filteredBoundaries[colIndex];
+ const endTime = filteredBoundaries[colIndex + 1];
+ const duration = endTime - startTime;
+
+ // Map duration to musical note values
+ let noteValue;
+ if (duration < 750) noteValue = 8; // 1/8 note (350-750ms)
+ else if (duration < 1500) noteValue = 4; // 1/4 note (750-1500ms)
+ else if (duration < 3000) noteValue = 2; // 1/2 note (1500-3000ms)
+ else noteValue = 1; // whole note (3000ms+)
+
+ // Collect all pitches that should sound during this time segment
+ const pitches = [];
+ let hasNonGreenColor = false;
+
+ this.colorData.forEach((rowData, rowIndex) => {
+ if (rowData.note && this._hasNonGreenColorInTimeRange(rowData, startTime, endTime)) {
+ const pitchInfo = this._convertRowToPitch(rowData);
+ if (pitchInfo) {
+ pitches.push(pitchInfo);
+ hasNonGreenColor = true;
+ }
+ }
+ });
+
+ // Add note or rest to the musical sequence
+ this._notesToPlay.push({
+ pitches: pitches,
+ noteValue: noteValue,
+ duration: duration,
+ isRest: !hasNonGreenColor || pitches.length === 0
+ });
+ }
+};
+
+this._filterSmallSegments = function(boundaries) {
+ const minDuration = 350; // Minimum duration for valid musical segments
+ const filteredBoundaries = [boundaries[0]];
+
+ for (let i = 1; i < boundaries.length; i++) {
+ const segmentDuration = boundaries[i] - filteredBoundaries[filteredBoundaries.length - 1];
+
+ // Only keep boundaries that create segments meeting minimum duration
+ if (segmentDuration >= minDuration) {
+ filteredBoundaries.push(boundaries[i]);
+ }
+ // Small segments are absorbed into adjacent larger segments
+ }
+
+ return filteredBoundaries;
+};
+```
+
+### Interactive Control System
+
+The widget provides comprehensive controls for fine-tuning the scanning process:
+
+```javascript
+this.createZoomControls = function() {
+ this.zoomControls = document.createElement("div");
+ this.zoomControls.style.position = "absolute";
+ this.zoomControls.style.bottom = "0";
+ this.zoomControls.style.display = "flex";
+ this.zoomControls.style.alignItems = "center";
+ this.zoomControls.style.gap = "8px";
+ this.zoomControls.style.backgroundColor = "#f0f0f0";
+ this.zoomControls.style.padding = "10px";
+ this.zoomControls.style.borderTop = "1px solid #888";
+
+ // Zoom control slider (0.1x to 3.0x magnification)
+ this.zoomSlider = document.createElement("input");
+ this.zoomSlider.type = "range";
+ this.zoomSlider.min = "0.1";
+ this.zoomSlider.max = "3";
+ this.zoomSlider.step = "0.01";
+ this.zoomSlider.value = "1";
+ this.zoomSlider.oninput = () => this._handleZoom();
+
+ // Column spacing control (20px to 200px spacing)
+ this.spacingSlider = document.createElement("input");
+ this.spacingSlider.type = "range";
+ this.spacingSlider.min = "20";
+ this.spacingSlider.max = "200";
+ this.spacingSlider.step = "1";
+ this.spacingSlider.value = "50";
+ this.spacingSlider.oninput = () => this._handleVerticalSpacing();
+
+ // Fine adjustment buttons for precise control
+ const zoomOut = document.createElement("button");
+ zoomOut.textContent = "−";
+ zoomOut.onclick = () => this._adjustZoom(-0.01);
+
+ const zoomIn = document.createElement("button");
+ zoomIn.textContent = "+";
+ zoomIn.onclick = () => this._adjustZoom(0.01);
+};
+
+this._handleZoom = function() {
+ if (this.imageWrapper) {
+ this.currentZoom = parseFloat(this.zoomSlider.value);
+ this.imageWrapper.style.transform = `scale(${this.currentZoom})`;
+ this.zoomValue.textContent = Math.round(this.currentZoom * 100) + '%';
+
+ // Redraw grid lines to match new zoom level
+ setTimeout(() => this._drawGridLines(), 50);
+ }
+};
+```
+
+### Grid Overlay and Visual Feedback System
+
+The widget provides real-time visual feedback through an overlay grid system:
+
+```javascript
+this._drawGridLines = function() {
+ if (!this.rowHeaderTable.rows.length || !this.gridOverlay) return;
+
+ this.gridOverlay.innerHTML = '';
+ const numRows = this.matrixData.rows.length;
+
+ // Draw horizontal red lines for each musical row
+ for (let i = 0; i < numRows; i++) {
+ const line = document.createElement('div');
+ line.style.position = 'absolute';
+ line.style.left = '0px';
+ line.style.right = '0px';
+ line.style.height = '2px';
+ line.style.backgroundColor = 'red';
+ line.style.zIndex = '5';
+
+ const position = (i + 1) * ROW_HEIGHT;
+ line.style.top = `${position}px`;
+
+ this.gridOverlay.appendChild(line);
+ }
+
+ // Draw vertical blue lines for timing columns
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+ const overlayWidth = overlayRect.width || 800;
+ const numVerticalLines = Math.floor(overlayWidth / this.verticalSpacing);
+
+ for (let i = 1; i <= numVerticalLines; i++) {
+ const verticalLine = document.createElement('div');
+ verticalLine.style.position = 'absolute';
+ verticalLine.style.top = '0px';
+ verticalLine.style.bottom = '0px';
+ verticalLine.style.width = '1px';
+ verticalLine.style.backgroundColor = 'blue';
+ verticalLine.style.left = `${i * this.verticalSpacing}px`;
+
+ this.gridOverlay.appendChild(verticalLine);
+ }
+};
+```
+
+### Animation and Scanning System
+
+The scanning animation system creates moving vertical lines that analyze color patterns:
+
+```javascript
+this._playPhrase = function() {
+ this._stopPlayback();
+ this.activity.textMsg(_("Scanning image with vertical lines..."));
+
+ // Create scanning lines for each musical note row
+ this.scanningLines = [];
+ this.colorData = [];
+
+ // Initialize scanning line for each note row
+ this.matrixData.rows.forEach((row, index) => {
+ if (row.note) { // Only create lines for musical notes
+ const scanLine = document.createElement('div');
+ scanLine.style.position = 'absolute';
+ scanLine.style.width = '2px';
+ scanLine.style.backgroundColor = '#FF0000';
+ scanLine.style.zIndex = '15';
+ scanLine.style.left = '0px';
+
+ // Position line within the row bounds
+ const topPos = index * ROW_HEIGHT;
+ const bottomPos = (index + 1) * ROW_HEIGHT;
+ scanLine.style.top = `${topPos}px`;
+ scanLine.style.height = `${ROW_HEIGHT}px`;
+
+ this.gridOverlay.appendChild(scanLine);
+
+ // Track line properties for animation
+ this.scanningLines.push({
+ element: scanLine,
+ currentX: 0,
+ rowIndex: index,
+ topPos: topPos,
+ bottomPos: bottomPos,
+ currentColor: null,
+ lastColorTime: performance.now()
+ });
+
+ // Initialize color data storage
+ this.colorData.push({
+ note: row.note,
+ colorSegments: []
+ });
+ }
+ });
+
+ // Start animation
+ this.isPlaying = true;
+ this.startTime = performance.now();
+ this.lastFrameTime = this.startTime;
+ this._animateLines();
+};
+
+this._animateLines = function() {
+ if (!this.isPlaying) return;
+
+ const now = performance.now();
+ const deltaTime = (now - this.lastFrameTime) / 1000;
+ this.lastFrameTime = now;
+
+ // Calculate scan speed (500ms between column markers)
+ const timeBetweenColumns = 0.5;
+ const scanSpeed = this.verticalSpacing / timeBetweenColumns;
+
+ let allLinesCompleted = true;
+
+ // Update each scanning line
+ this.scanningLines.forEach(line => {
+ if (line.completed) return;
+
+ // Move line horizontally
+ line.currentX += scanSpeed * deltaTime;
+ line.element.style.left = `${line.currentX}px`;
+
+ // Sample color at current position
+ this._sampleAndDetectColor(line, now);
+
+ // Check if line has completed scanning
+ if (this._isLineBeyondImageHorizontally(line)) {
+ line.completed = true;
+ line.element.style.display = 'none';
+ } else {
+ allLinesCompleted = false;
+ }
+ });
+
+ if (allLinesCompleted) {
+ this._stopPlayback();
+ } else {
+ requestAnimationFrame(() => this._animateLines());
+ }
+};
+```
+
+### Audio Synthesis and Playback System
+
+The widget includes a sophisticated audio system for real-time note playback:
+
+```javascript
+this._initAudio = function() {
+ // Create synthesizer instance with piano samples
+ this.synth = new Synth();
+ this.synth.loadSamples();
+
+ // Configure piano synthesizer for all pitch playback
+ this.synth.createSynth(0, "piano", "piano", null);
+};
+
+this._playNote = function(note, duration = 0.5) {
+ if (!this.synth) return;
+
+ try {
+ // Trigger note playback with specified parameters
+ this.synth.trigger(0, note, duration, "piano", null, null, false, 0);
+ } catch (e) {
+ console.error("Error playing note:", e);
+ this.activity.textMsg(_("Audio playback error: ") + e.message);
+ }
+};
+
+// Polyphonic playback system for multiple simultaneous notes
+this.playColorMusicPolyphonic = async function(colorData) {
+ if (!this.synth) return;
+
+ // Use filtered boundaries for consistent timing
+ const columnBoundaries = this._analyzeColumnBoundaries();
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ let events = [];
+
+ // Build timeline of note events
+ for (let colIndex = 0; colIndex < filteredBoundaries.length - 1; colIndex++) {
+ const startTime = filteredBoundaries[colIndex];
+ const endTime = filteredBoundaries[colIndex + 1];
+ const duration = endTime - startTime;
+
+ // Check each row for notes to play
+ this.colorData.forEach((rowData, rowIndex) => {
+ if (rowData.note && this._hasNonGreenColorInTimeRange(rowData, startTime, endTime)) {
+ events.push({
+ type: 'noteOn',
+ time: startTime,
+ note: rowData.note,
+ duration: duration
+ });
+
+ events.push({
+ type: 'noteOff',
+ time: startTime + duration,
+ note: rowData.note
+ });
+ }
+ });
+ }
+
+ // Play events in chronological order
+ events.sort((a, b) => a.time - b.time);
+
+ let playingNotes = new Set();
+ let lastTime = 0;
+
+ for (let i = 0; i < events.length; i++) {
+ const event = events[i];
+ const delay = Math.max(0, event.time - lastTime);
+
+ if (delay > 0) {
+ await new Promise(resolve => setTimeout(resolve, delay));
+ }
+
+ if (event.type === 'noteOn') {
+ this._playNote(event.note, event.duration / 1000);
+ playingNotes.add(event.note);
+ } else {
+ playingNotes.delete(event.note);
+ }
+
+ lastTime = event.time;
+ }
+};
+```
+
+### Export System and Music Blocks Integration
+
+The widget creates proper Music Blocks action blocks from detected musical patterns:
+
+```javascript
+this._savePhrase = function() {
+ if (!this.colorData || this.colorData.length === 0) {
+ this.activity.textMsg(_("No color data to save. Please scan an image first."));
+ return;
+ }
+
+ // Collect and analyze musical notes from color data
+ this._collectNotesToPlay();
+
+ if (this._notesToPlay.length === 0) {
+ this.activity.textMsg(_("No notes detected from color scanning."));
+ return;
+ }
+
+ // Hide palettes during block creation
+ for (const name in this.activity.blocks.palettes.dict) {
+ this.activity.blocks.palettes.dict[name].hideMenu(true);
+ }
+ this.activity.refreshCanvas();
+
+ // Create Music Blocks action block structure
+ const newStack = [
+ [0, ["action", { collapsed: true }], 100, 100, [null, 1, null, null]],
+ [1, ["text", { value: _("LEGO phrase") }], 0, 0, [0]]
+ ];
+ let endOfStackIdx = 0;
+
+ // Convert each detected note into Music Blocks note blocks
+ for (let i = 0; i < this._notesToPlay.length; i++) {
+ const note = this._notesToPlay[i];
+ const idx = newStack.length;
+
+ // Create new note block
+ newStack.push([idx, "newnote", 0, 0, [endOfStackIdx, idx + 1, idx + 2, null]]);
+
+ // Set up block connections
+ if (i === 0) {
+ newStack[0][4][2] = idx; // Connect to action block
+ } else {
+ newStack[endOfStackIdx][4][3] = idx; // Connect to previous note
+ }
+ endOfStackIdx = idx;
+
+ // Add note duration as fraction
+ const delta = 5;
+ newStack.push([idx + 1, "vspace", 0, 0, [idx, idx + delta]]);
+
+ // Convert note value to fraction format
+ let numerator, denominator;
+ if (note.noteValue === 1.5) {
+ numerator = 3; denominator = 2; // dotted half note
+ } else {
+ numerator = 1; denominator = note.noteValue;
+ }
+
+ newStack.push([idx + 2, "divide", 0, 0, [idx, idx + 3, idx + 4]]);
+ newStack.push([idx + 3, ["number", { value: numerator }], 0, 0, [idx + 2]]);
+ newStack.push([idx + 4, ["number", { value: denominator }], 0, 0, [idx + 2]]);
+
+ // Connect note duration blocks
+ newStack[idx][4][1] = idx + 2; // divide block
+ newStack[idx][4][2] = idx + 1; // vspace block
+
+ // Add pitch information for non-rest notes
+ if (!note.isRest && note.pitches.length > 0) {
+ let lastConnection = idx + 1; // vspace block
+
+ note.pitches.forEach((pitch, pitchIndex) => {
+ const pitchIdx = newStack.length;
+
+ // Create pitch block with solfege and octave
+ newStack.push([pitchIdx, "pitch", 0, 0, [lastConnection, pitchIdx + 1, pitchIdx + 2, null]]);
+ newStack.push([pitchIdx + 1, ["solfege", { value: pitch.solfege }], 0, 0, [pitchIdx]]);
+ newStack.push([pitchIdx + 2, ["number", { value: pitch.octave }], 0, 0, [pitchIdx]]);
+
+ // Update connection chain
+ if (lastConnection !== null) {
+ newStack[lastConnection][4][newStack[lastConnection][4].length - 1] = pitchIdx;
+ }
+ lastConnection = pitchIdx;
+ });
+ }
+ }
+
+ // Load the completed block structure into Music Blocks
+ this.activity.blocks.loadNewBlocks(newStack);
+ this.activity.textMsg(_("LEGO phrase saved as action blocks with ") + this._notesToPlay.length + _(" notes"));
+};
+```
+
+### Data Visualization and Debugging
+
+The widget generates visual representations of detected color patterns for analysis:
+
+```javascript
+this._generateColorVisualization = function() {
+ if (!this.colorData || this.colorData.length === 0) return;
+
+ // Create high-resolution canvas for visualization
+ const canvasWidth = 800;
+ const rowHeight = 50;
+ const canvasHeight = this.colorData.length * rowHeight;
+
+ const canvas = document.createElement('canvas');
+ canvas.width = canvasWidth;
+ canvas.height = canvasHeight;
+ const ctx = canvas.getContext('2d');
+
+ // Color mapping for visualization
+ const colorMap = {
+ 'red': '#FF0000', 'orange': '#FFA500', 'yellow': '#FFFF00',
+ 'green': '#00FF00', 'blue': '#0000FF', 'purple': '#800080',
+ 'pink': '#FFC0CB', 'cyan': '#00FFFF', 'magenta': '#FF00FF',
+ 'white': '#FFFFFF', 'black': '#000000', 'gray': '#808080',
+ 'unknown': '#C0C0C0'
+ };
+
+ // Draw background
+ ctx.fillStyle = '#f0f0f0';
+ ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+
+ // Render each row's color segments
+ this.colorData.forEach((rowData, rowIndex) => {
+ const y = rowIndex * rowHeight;
+
+ // Draw row label
+ ctx.fillStyle = '#000000';
+ ctx.font = 'bold 12px Arial';
+ ctx.textAlign = 'left';
+ ctx.fillText(`${rowData.note || 'Unknown'}`, 10, y + 25);
+
+ // Draw color segments
+ if (rowData.colorSegments && rowData.colorSegments.length > 0) {
+ let currentX = 150; // Start after label area
+ const availableWidth = canvasWidth - 150 - 20;
+
+ // Calculate total duration for proportional sizing
+ const totalDuration = rowData.colorSegments.reduce((sum, seg) => sum + seg.duration, 0);
+
+ rowData.colorSegments.forEach(segment => {
+ const segmentWidth = (segment.duration / totalDuration) * availableWidth;
+
+ ctx.fillStyle = colorMap[segment.color] || colorMap.unknown;
+ ctx.fillRect(currentX, y + 5, segmentWidth, rowHeight - 10);
+
+ // Add segment border
+ ctx.strokeStyle = '#333333';
+ ctx.lineWidth = 1;
+ ctx.strokeRect(currentX, y + 5, segmentWidth, rowHeight - 10);
+
+ currentX += segmentWidth;
+ });
+ }
+ });
+
+ // Add column boundary lines
+ this._drawColumnLines(ctx, canvasWidth, canvasHeight, 150, canvasWidth - 170);
+
+ // Export as downloadable PNG
+ canvas.toBlob((blob) => {
+ const url = URL.createObjectURL(blob);
+ const a = document.createElement('a');
+ a.href = url;
+ a.download = 'lego-bricks-color-analysis.png';
+ document.body.appendChild(a);
+ a.click();
+ document.body.removeChild(a);
+ URL.revokeObjectURL(url);
+ }, 'image/png');
+};
+```
+
+### Usage Workflow and Best Practices
+
+1. **Widget Initialization**: Open the LEGO Bricks widget from Music Blocks after connecting pitch blocks
+2. **Content Loading**: Upload a high-contrast image or start webcam with clear color patterns
+3. **Image Positioning**: Use drag functionality to position the image optimally within the scanning area
+4. **Settings Adjustment**: Fine-tune zoom (0.1x to 3.0x) and column spacing (20px to 200px) for optimal scanning
+5. **Scanning Process**: Click the play button to initiate vertical line scanning with real-time color detection
+6. **Results Analysis**: Review the generated color visualization PNG and listen to polyphonic playback
+7. **Musical Export**: Save the detected musical phrase as Music Blocks action blocks for further editing
+
+### Technical Requirements and Browser Compatibility
+
+- **WebRTC Support**: Essential for webcam functionality in modern browsers
+- **HTML5 Canvas API**: Required for pixel-level image analysis and color detection
+- **Web Audio API**: Needed for real-time audio synthesis and note playback
+- **File API**: Necessary for image upload and PNG export functionality
+- **ES6+ JavaScript**: Modern JavaScript features for optimal performance
+- **Minimum Browser Versions**: Chrome 60+, Firefox 55+, Safari 11+, Edge 79+
+
+### Performance Optimization Features
+
+- **Efficient Color Sampling**: 32-point vertical sampling reduces computational overhead
+- **Temporal Filtering**: 350ms minimum duration eliminates noise and improves musical quality
+- **Canvas Optimization**: Temporary canvas creation and disposal prevents memory leaks
+- **Animation Frame Management**: RequestAnimationFrame ensures smooth 60fps scanning animation
+- **Memory Management**: Automatic cleanup of scanning lines and color data after processing
+
+### Accessibility and Inclusive Design
+
+The LEGO Bricks widget incorporates comprehensive accessibility features:
+
+- **Visual-to-Audio Conversion**: Transforms visual content into musical sequences for visually impaired users
+- **High Contrast Interface**: Red horizontal lines and blue vertical lines provide clear visual guidance
+- **Keyboard Navigation**: All controls accessible via keyboard shortcuts and tab navigation
+- **Screen Reader Compatibility**: Semantic HTML structure with proper ARIA labels
+- **Color-blind Accessibility**: Uses multiple visual cues including position, timing, and contrast
+- **Customizable Interface**: Adjustable zoom and spacing accommodate different visual needs
+- **Real-time Feedback**: Audio confirmation of detected colors and musical notes
+- **Error Handling**: Comprehensive error messages guide users through troubleshooting
+
## Code of Conduct
The Music Blocks project adheres to the [Sugar Labs Code of
diff --git a/css/activities.css b/css/activities.css
index 7d43b46176..6ab1e09b93 100644
--- a/css/activities.css
+++ b/css/activities.css
@@ -2021,6 +2021,8 @@ table {
max-width: 80%;
}
+
+
.chatInterface{
display: flex;
flex-direction: column;
@@ -2070,3 +2072,17 @@ table {
align-self: flex-end;
background-color: #DCF8C6;
}
+
+
+.lego-brick {
+ display: inline-block;
+ background-color: #FF0000;
+ border: 1px solid #880000;
+ margin: 2px;
+}
+
+.lego-size-1 { width: 20px; height: 10px; }
+.lego-size-2 { width: 40px; height: 10px; }
+/* ... more sizes ... */
+
+
diff --git a/css/style.css b/css/style.css
index 1c6d4193c3..216a5f565b 100644
--- a/css/style.css
+++ b/css/style.css
@@ -99,3 +99,15 @@ input[type="range"]:focus::-ms-fill-lower {
input[type="range"]:focus::-ms-fill-upper {
background: #90c100;
}
+
+.lego-brick {
+ display: inline-block;
+ background-color: #FF0000;
+ border: 1px solid #880000;
+ margin: 2px;
+}
+
+.lego-size-1 { width: 20px; height: 10px; }
+.lego-size-2 { width: 40px; height: 10px; }
+/* ... more sizes ... */
+
diff --git a/header-icons/upload-button.svg b/header-icons/upload-button.svg
new file mode 100644
index 0000000000..902fdf7d85
--- /dev/null
+++ b/header-icons/upload-button.svg
@@ -0,0 +1,5 @@
+
diff --git a/header-icons/webcam-button.svg b/header-icons/webcam-button.svg
new file mode 100644
index 0000000000..46013f8e58
--- /dev/null
+++ b/header-icons/webcam-button.svg
@@ -0,0 +1,3 @@
+
diff --git a/js/activity.js b/js/activity.js
index 00272dfb8f..426db7bb88 100644
--- a/js/activity.js
+++ b/js/activity.js
@@ -174,6 +174,7 @@ if (_THIS_IS_MUSIC_BLOCKS_) {
"widgets/oscilloscope",
"widgets/sampler",
"widgets/reflection",
+ "widgets/legobricks",
"activity/lilypond",
"activity/abc",
"activity/midi",
diff --git a/js/block.js b/js/block.js
index a584f125ce..ff125a27b6 100644
--- a/js/block.js
+++ b/js/block.js
@@ -4246,6 +4246,7 @@ class Block {
case "pitch staircase":
case "status":
case "phrase maker":
+ case "lego bricks":
case "custom mode":
case "music keyboard":
case "pitch drum":
diff --git a/js/blocks.js b/js/blocks.js
index d980a8c05c..5c684c5661 100644
--- a/js/blocks.js
+++ b/js/blocks.js
@@ -1579,6 +1579,7 @@ class Blocks {
case "pitch staircase":
case "status":
case "phrase maker":
+ case "lego bricks":
case "custom mode":
case "music keyboard":
case "pitch drum":
@@ -2035,6 +2036,7 @@ class Blocks {
case "pitch staircase":
case "status":
case "phrase maker":
+ case "lego bricks":
case "custom mode":
case "music keyboard":
case "pitch drum":
diff --git a/js/blocks/PitchBlocks.js b/js/blocks/PitchBlocks.js
index 6a9898f444..3465e4ad66 100644
--- a/js/blocks/PitchBlocks.js
+++ b/js/blocks/PitchBlocks.js
@@ -1614,6 +1614,17 @@ function setupPitchBlocks(activity) {
// convert hertz to note/octave
const note = obj;
tur.singer.lastNotePlayed = [note[0] + note[1], 4];
+ } else if (logo.inLegoWidget) {
+ logo.legoWidget.addRowBlock(blk);
+ if (!logo.pitchBlocks.includes(blk)) {
+ logo.pitchBlocks.push(blk);
+ }
+
+ logo.legoWidget.rowLabels.push(activity.blocks.blockList[blk].name);
+ logo.legoWidget.rowArgs.push(arg.toFixed(0));
+ // convert hertz to note/octave
+ const note = obj;
+ tur.singer.lastNotePlayed = [note[0] + note[1], 4];
} else if (logo.inMusicKeyboard) {
logo.musicKeyboard.instruments.push(last(tur.singer.instrumentNames));
logo.musicKeyboard.noteNames.push("hertz");
diff --git a/js/blocks/RhythmBlockPaletteBlocks.js b/js/blocks/RhythmBlockPaletteBlocks.js
index 6ac3de7f82..1245c674e7 100644
--- a/js/blocks/RhythmBlockPaletteBlocks.js
+++ b/js/blocks/RhythmBlockPaletteBlocks.js
@@ -112,6 +112,11 @@ function setupRhythmBlockPaletteBlocks(activity) {
if (logo.inMatrix || logo.tuplet) {
if (logo.inMatrix) {
logo.phraseMaker.addColBlock(blk, arg0);
+
+ // Add individual entries for each beat to avoid extra × blocks
+ for (let i = 0; i < arg0; i++) {
+ logo.tupletRhythms.push(["individual", 1, noteBeatValue]);
+ }
}
for (let i = 0; i < args[0]; i++) {
diff --git a/js/blocks/WidgetBlocks.js b/js/blocks/WidgetBlocks.js
index c2c350afe5..06bbf6ac64 100644
--- a/js/blocks/WidgetBlocks.js
+++ b/js/blocks/WidgetBlocks.js
@@ -19,7 +19,7 @@
RhythmRuler, FILTERTYPES, instrumentsFilters, DEFAULTFILTERTYPE,
TemperamentWidget, TimbreWidget, ModeWidget, PitchSlider,
MusicKeyboard, PitchStaircase, SampleWidget, _THIS_IS_MUSIC_BLOCKS_,
- Arpeggio
+ Arpeggio, LegoWidget
*/
/*
@@ -60,8 +60,10 @@
MusicKeyboard
- js/widgets/pitchstaircase.js
PitchStaircase
- - js/widgets/aidebugger.js
+ - js/widgets/aidebugger.js
AIDebuggerWidget
+ - js/widgets/legobricks.js
+ LegoWidget
*/
@@ -1639,8 +1641,7 @@ function setupWidgetBlocks(activity) {
return [args[0], 1];
}
}
-
-
+
class ReflectionBlock extends StackClampBlock {
/**
* Creates a ReflectionBlock instance.
@@ -1695,7 +1696,93 @@ function setupWidgetBlocks(activity) {
}
}
-
+
+ /**
+ * Represents a block for controlling LEGO brick parameters and visualization.
+ * @extends StackClampBlock
+ */
+ class LegoBricksBlock extends StackClampBlock {
+ constructor() {
+ super("legobricks");
+ this.setPalette("widgets", activity);
+ this.parameter = true;
+ this.beginnerBlock(true);
+
+ this.setHelpString([
+ _("The LEGO Bricks block opens a widget for designing virtual LEGO creations."),
+ "documentation",
+ null,
+ "legobricks"
+ ]);
+
+ //.TRANS: LEGO bricks designer
+ this.formBlock({ name: _("LEGO Bricks"), canCollapse: true });
+ this.makeMacro((x, y) => [
+ [0, "legobricks", x, y, [null, 1, 18]],
+ [1, "pitch", 0, 0, [0, 2, 3, 4]],
+ [2, ["solfege", { value: "do" }], 0, 0, [1]],
+ [3, ["number", { value: 4 }], 0, 0, [1]],
+ [4, "pitch", 0, 0, [1, 5, 6, 7]],
+ [5, ["solfege", { value: "re" }], 0, 0, [4]],
+ [6, ["number", { value: 4 }], 0, 0, [4]],
+ [7, "pitch", 0, 0, [4, 8, 9, 10]],
+ [8, ["solfege", { value: "mi" }], 0, 0, [7]],
+ [9, ["number", { value: 4 }], 0, 0, [7]],
+ [10, "pitch", 0, 0, [7, 11, 12, 13]],
+ [11, ["solfege", { value: "fa" }], 0, 0, [10]],
+ [12, ["number", { value: 4 }], 0, 0, [10]],
+ [13, "pitch", 0, 0, [10, 14, 15, 16]],
+ [14, ["solfege", { value: "sol" }], 0, 0, [13]],
+ [15, ["number", { value: 4 }], 0, 0, [13]],
+ [16, "playdrum", 0, 0, [13, 17, null]],
+ [17, ["drumname", { value: "kick drum" }], 0, 0, [16]],
+ [18, "hiddennoflow", 0, 0, [0, null]]
+ ]);
+ }
+
+ /**
+ * Handles the flow of data for the LEGO bricks block.
+ * @param {any[]} args - The arguments passed to the block.
+ * @param {object} logo - The logo object.
+ * @param {object} turtle - The turtle object.
+ * @param {object} blk - The block object.
+ * @returns {number[]} - The output values.
+ */
+ flow(args, logo, turtle, blk) {
+ logo.inLegoWidget = true;
+
+ if (logo.legoWidget === null) {
+ logo.legoWidget = new LegoWidget();
+ }
+ logo.legoWidget.blockNo = blk;
+
+ logo.legoWidget.rowLabels = [];
+ logo.legoWidget.rowArgs = [];
+ logo.legoWidget.clearBlocks();
+
+ const listenerName = "_legobricks_" + turtle;
+ logo.setDispatchBlock(blk, turtle, listenerName);
+
+ const __listener = () => {
+ if (logo.legoWidget.rowLabels.length === 0) {
+ activity.errorMsg(
+ _("You must have at least one pitch block in the LEGO bricks widget."),
+ blk
+ );
+ } else {
+ logo.legoWidget.blockNo = blk;
+ logo.legoWidget.init(activity);
+ }
+ logo.inLegoWidget = false;
+ };
+
+ logo.setTurtleListener(turtle, listenerName, __listener);
+
+ if (args.length === 1) return [args[0], 1];
+ }
+ }
+
+
class AIDebugger extends StackClampBlock {
constructor() {
super("aidebugger");
@@ -1745,8 +1832,6 @@ function setupWidgetBlocks(activity) {
return [args[0], 1];
}
}
-
-
// Set up blocks if this is Music Blocks environment
if (_THIS_IS_MUSIC_BLOCKS_) {
new EnvelopeBlock().setup(activity);
@@ -1762,6 +1847,7 @@ function setupWidgetBlocks(activity) {
new oscilloscopeWidgetBlock().setup(activity);
new PitchSliderBlock().setup(activity);
new ChromaticBlock().setup(activity);
+ new LegoBricksBlock().setup(activity);
new ReflectionBlock().setup(activity);
// new AIMusicBlocks().setup(activity);
new MusicKeyboard2Block().setup(activity);
diff --git a/js/logo.js b/js/logo.js
index faf88c94a9..79057aa3be 100644
--- a/js/logo.js
+++ b/js/logo.js
@@ -112,6 +112,7 @@ class Logo {
// Widgets
this.reflection = null;
this.phraseMaker = null;
+ this.legoWidget = null;
this.pitchDrumMatrix = null;
this.arpeggio = null;
this.rhythmRuler = null;
@@ -126,6 +127,7 @@ class Logo {
this.oscilloscopeTurtles = [];
this.meterWidget = null;
this.statusMatrix = null;
+ this.legobricks = null;
this.evalFlowDict = {};
this.evalArgDict = {};
@@ -184,6 +186,7 @@ class Logo {
// pitch-rhythm matrix
this.inMatrix = false;
+ this.inLegoWidget = false;
this.tupletRhythms = [];
this.addingNotesToTuplet = false;
this.drumBlocks = [];
@@ -1064,6 +1067,7 @@ class Logo {
this.inPitchDrumMatrix = false;
this.inMatrix = false;
+ this.inLegoWidget = false;
this.inMusicKeyboard = false;
this.inTimbre = false;
this.inArpeggio = false;
diff --git a/js/turtle-singer.js b/js/turtle-singer.js
index f74a34e9a0..bbf1d64993 100644
--- a/js/turtle-singer.js
+++ b/js/turtle-singer.js
@@ -233,6 +233,14 @@ class Singer {
logo.phraseMaker.rowLabels.push(activity.logo.blocks.blockList[blk].name);
logo.phraseMaker.rowArgs.push(args[0]);
+ } else if (logo.inLegoWidget && !logo.inMatrix) {
+ logo.legoWidget.addRowBlock(blk);
+ if (!logo.pitchBlocks.includes(blk)) {
+ logo.pitchBlocks.push(blk);
+ }
+
+ logo.legoWidget.rowLabels.push(activity.logo.blocks.blockList[blk].name);
+ logo.legoWidget.rowArgs.push(args[0]);
} else if (logo.inPitchSlider) {
logo.pitchSlider.frequency = args[0];
} else {
@@ -910,6 +918,56 @@ class Singer {
activity.logo.phraseMaker.rowArgs.push(noteObj[1]);
}
}
+ } else if (activity.logo.inLegoWidget && !activity.logo.inMatrix) {
+ if (note.toLowerCase() !== "rest") {
+ activity.logo.legoWidget.addRowBlock(blk);
+ if (!activity.logo.pitchBlocks.includes(blk)) {
+ activity.logo.pitchBlocks.push(blk);
+ }
+ }
+
+ const duplicateFactor =
+ tur.singer.duplicateFactor.length > 0 ? tur.singer.duplicateFactor : 1;
+
+ for (let i = 0; i < duplicateFactor; i++) {
+ // Apply transpositions
+ const transposition = 2 * delta + tur.singer.transposition;
+ const alen = tur.singer.arpeggio.length;
+ let atrans = transposition + cents;
+ if (alen > 0 && i < alen) {
+ atrans += tur.singer.arpeggio[i];
+ }
+ const noteObj = getNote(
+ note,
+ octave,
+ atrans, // transposition,
+ tur.singer.keySignature,
+ tur.singer.movable,
+ null,
+ activity.errorMsg,
+ activity.logo.synth.inTemperament
+ );
+ tur.singer.previousNotePlayed = tur.singer.lastNotePlayed;
+ tur.singer.lastNotePlayed = [noteObj[0] + noteObj[1], 4];
+
+ if (
+ tur.singer.keySignature[0] === "C" &&
+ tur.singer.keySignature[1].toLowerCase() === "major" &&
+ noteIsSolfege(note)
+ ) {
+ noteObj[0] = getSolfege(noteObj[0]);
+ }
+
+ // If we are in a setdrum clamp, override the pitch.
+ if (tur.singer.drumStyle.length > 0) {
+ activity.logo.legoWidget.rowLabels.push(last(tur.singer.drumStyle));
+ activity.logo.legoWidget.rowArgs.push(-1);
+ } else {
+ // Don't bother with the name conversions.
+ activity.logo.legoWidget.rowLabels.push(noteObj[0]);
+ activity.logo.legoWidget.rowArgs.push(noteObj[1]);
+ }
+ }
} else if (tur.singer.inNoteBlock.length > 0) {
// maybe of interest
tur.singer.inverted = tur.singer.invertList.length > 0;
@@ -1479,6 +1537,13 @@ class Singer {
);
}
}
+ // Note: tupletRhythms will be populated by the rhythm blocks themselves
+ } else if (activity.logo.inLegoWidget && !activity.logo.inMatrix) {
+ // For LEGO widget, we don't need to handle rhythm blocks currently
+ // Just store the note information
+ if (tur.singer.inNoteBlock.length > 0) {
+ // Could add timing information here if needed in the future
+ }
noteBeatValue *= tur.singer.beatFactor;
if (activity.logo.tuplet) {
diff --git a/js/widgets/legobricks.js b/js/widgets/legobricks.js
new file mode 100644
index 0000000000..4d2f8e59ac
--- /dev/null
+++ b/js/widgets/legobricks.js
@@ -0,0 +1,2266 @@
+/*
+ exported LegoWidget
+*/
+
+/**
+ * Represents a LEGO Bricks Widget with Phrase Maker functionality.
+ * @constructor
+ */
+function LegoWidget() {
+ const ICONSIZE = 32;
+ const WIDGETWIDTH = 1200;
+ const WIDGETHEIGHT = 700;
+ const ROW_HEIGHT = 40; // Fixed row height for both matrix and image canvas
+
+ // Matrix data structure with pitch mappings
+ this.matrixData = {
+ rows: [
+ { type: "pitch", label: "High C (Do)", icon: "HighC.png", color: "pitch-row", note: "C5" },
+ { type: "pitch", label: "B (Ti)", icon: "B.png", color: "pitch-row", note: "B4" },
+ { type: "pitch", label: "A (La)", icon: "A.png", color: "pitch-row", note: "A4" },
+ { type: "pitch", label: "G (So)", icon: "G.png", color: "pitch-row", note: "G4" },
+ { type: "pitch", label: "F (Fa)", icon: "F.png", color: "pitch-row", note: "F4" },
+ { type: "pitch", label: "E (Mi)", icon: "E.png", color: "pitch-row", note: "E4" },
+ { type: "pitch", label: "D (Re)", icon: "D.png", color: "pitch-row", note: "D4" },
+ { type: "pitch", label: "Middle C (Do)", icon: "MiddleC.png", color: "pitch-row", note: "C4" },
+ { type: "pitch", label: "B (Low Ti)", icon: "LowB.png", color: "pitch-row", note: "B3" },
+ { type: "pitch", label: "Low C (Low Do)", icon: "LowC.png", color: "pitch-row", note: "C3" },
+ { type: "pitch", label: "Zoom Controls", icon: "LowC.png", color: "pitch-row" }
+ ],
+ columns: 8,
+ selectedCells: new Set()
+ };
+
+ // Widget properties
+ this.widgetWindow = null;
+ this.activity = null;
+ this.isPlaying = false;
+ this.isDragging = false;
+ this.currentZoom = 1;
+ this.verticalSpacing = 50;
+ this.imageWrapper = null;
+ this.synth = null;
+ this.selectedInstrument = "electronic synth";
+
+ // Pitch block handling properties (similar to PhraseMaker)
+ this.blockNo = null;
+ this.rowLabels = [];
+ this.rowArgs = [];
+ this._rowBlocks = [];
+ this._rowMap = [];
+ this._rowOffset = [];
+ this._notesToPlay = []; // Array to store notes for action block export
+
+ /**
+ * Clears block references within the LegoWidget.
+ * Resets arrays used to track row blocks.
+ */
+ this.clearBlocks = function() {
+ this._rowBlocks = [];
+ this._rowMap = [];
+ this._rowOffset = [];
+ };
+
+ /**
+ * Adds a row block to the LegoWidget matrix.
+ * This method is called when encountering a pitch block during matrix creation.
+ * @param {number} rowBlock - The pitch block identifier to add to the matrix row.
+ */
+ this.addRowBlock = function(rowBlock) {
+ this._rowMap.push(this._rowBlocks.length);
+ this._rowOffset.push(0);
+ // In case there is a repeat block, use a unique block number
+ // for each instance.
+ while (this._rowBlocks.includes(rowBlock)) {
+ rowBlock = rowBlock + 1000000;
+ }
+ this._rowBlocks.push(rowBlock);
+ };
+
+ /**
+ * Generates rows based on the pitch blocks received from the LEGO bricks block.
+ * @private
+ */
+ this._generateRowsFromPitchBlocks = function() {
+ // Clear existing matrix data
+ this.matrixData.rows = [];
+
+ // Create a list of pitch entries for sorting
+ const pitchEntries = [];
+
+ // Generate rows based on the pitch blocks
+ for (let i = 0; i < this.rowLabels.length; i++) {
+ const pitchName = this.rowLabels[i];
+ const octave = this.rowArgs[i];
+
+ // Only process pitch blocks (skip drum blocks)
+ if (octave !== -1) {
+ // This is a pitch block
+ const noteLabel = pitchName + octave;
+
+ // Convert pitch to display name
+ let displayName = pitchName;
+ if (pitchName === "do") displayName = "Do";
+ else if (pitchName === "re") displayName = "Re";
+ else if (pitchName === "mi") displayName = "Mi";
+ else if (pitchName === "fa") displayName = "Fa";
+ else if (pitchName === "sol") displayName = "So";
+ else if (pitchName === "la") displayName = "La";
+ else if (pitchName === "ti") displayName = "Ti";
+ else displayName = pitchName.toUpperCase();
+
+ // Calculate frequency for sorting (same as phrasemaker)
+ let frequency = 0;
+ try {
+ if (typeof noteToFrequency !== "undefined") {
+ frequency = noteToFrequency(noteLabel, this.activity.turtles.ithTurtle(0).singer.keySignature);
+ } else {
+ // Fallback frequency calculation if noteToFrequency is not available
+ frequency = this._calculateFallbackFrequency(pitchName, octave);
+ }
+ } catch (e) {
+ // Fallback frequency calculation
+ frequency = this._calculateFallbackFrequency(pitchName, octave);
+ }
+
+ pitchEntries.push({
+ frequency: frequency,
+ type: "pitch",
+ label: displayName + " (" + octave + ")",
+ icon: "pitch.svg",
+ color: "pitch-row",
+ note: noteLabel,
+ pitch: pitchName,
+ octave: octave
+ });
+ }
+ }
+
+ // Sort pitch entries by frequency (highest first, like phrasemaker)
+ pitchEntries.sort((a, b) => b.frequency - a.frequency);
+
+ // Add sorted pitch entries to matrix data
+ this.matrixData.rows = pitchEntries;
+
+ // Add a control row at the end
+ this.matrixData.rows.push({
+ type: "control",
+ label: "Zoom Controls",
+ icon: "zoom.svg",
+ color: "control-row"
+ });
+
+ // If no pitch blocks were provided, add some default rows for testing (already sorted)
+ if (this.rowLabels.length === 0) {
+ this.matrixData.rows = [
+ { type: "pitch", label: "E4 (Mi)", icon: "pitch.svg", color: "pitch-row", note: "E4" },
+ { type: "pitch", label: "D4 (Re)", icon: "pitch.svg", color: "pitch-row", note: "D4" },
+ { type: "pitch", label: "C4 (Middle C)", icon: "pitch.svg", color: "pitch-row", note: "C4" },
+ { type: "control", label: "Zoom Controls", icon: "zoom.svg", color: "control-row" }
+ ];
+ }
+ };
+
+ /**
+ * Calculates frequency for a pitch name and octave as fallback when noteToFrequency is not available.
+ * @private
+ * @param {string} pitchName - The pitch name (e.g., "do", "C", "re", "D")
+ * @param {number} octave - The octave number
+ * @returns {number} The calculated frequency
+ */
+ this._calculateFallbackFrequency = function(pitchName, octave) {
+ // Handle both letter names and solfege
+ const noteFreqs = {
+ "C": 261.63, "do": 261.63,
+ "D": 293.66, "re": 293.66,
+ "E": 329.63, "mi": 329.63,
+ "F": 349.23, "fa": 349.23,
+ "G": 392.00, "sol": 392.00,
+ "A": 440.00, "la": 440.00,
+ "B": 493.88, "ti": 493.88
+ };
+
+ const baseFreq = noteFreqs[pitchName.toLowerCase()] || noteFreqs[pitchName.toUpperCase()] || noteFreqs["C"];
+ return baseFreq * Math.pow(2, octave - 4);
+ };
+
+ /**
+ * Initializes the LEGO Widget with Phrase Maker functionality.
+ * @param {object} activity - The activity object.
+ * @returns {void}
+ */
+ this.init = function(activity) {
+ this.activity = activity;
+ this.running = true;
+
+ // Initialize audio synthesizer
+ this._initAudio();
+
+ const widgetWindow = window.widgetWindows.windowFor(this, "LEGO BRICKS");
+ this.widgetWindow = widgetWindow;
+ widgetWindow.clear();
+ widgetWindow.show();
+
+ var that = this;
+
+ widgetWindow.onclose = () => {
+ this._stopWebcam();
+ this.running = false;
+ widgetWindow.destroy();
+ };
+
+ widgetWindow.onmaximize = this._scale.bind(this);
+
+ // Add control buttons in left sidebar
+ this.playButton = widgetWindow.addButton("play-button.svg", ICONSIZE, _("Play"));
+ this.playButton.onclick = () => {
+ this._playPhrase();
+ };
+
+ this.saveButton = widgetWindow.addButton("save-button.svg", ICONSIZE, _("Save"));
+ this.saveButton.onclick = () => {
+ this._savePhrase();
+ };
+
+ this.exportButton = widgetWindow.addButton("export-button.svg", ICONSIZE, _("Export"));
+ this.exportButton.onclick = () => {
+ this._exportPhrase();
+ };
+
+ this.uploadButton = widgetWindow.addButton("upload-button.svg", ICONSIZE, _("Upload Image"));
+ this.uploadButton.onclick = () => {
+ this._uploadImage();
+ };
+
+ this.webcamButton = widgetWindow.addButton("webcam-button.svg", ICONSIZE, _("Webcam"));
+ this.webcamButton.onclick = () => {
+ this._startWebcam();
+ };
+
+ this.clearButton = widgetWindow.addButton("clear-button.svg", ICONSIZE, _("Clear"));
+ this.clearButton.onclick = () => {
+ this._clearPhrase();
+ };
+
+ // Create main container
+ this.createMainContainer();
+
+ widgetWindow.sendToCenter();
+ this.widgetWindow = widgetWindow;
+
+ // Generate rows based on pitch blocks
+ this._generateRowsFromPitchBlocks();
+
+ // Re-initialize row headers with the dynamic rows
+ this._initializeRowHeaders();
+
+ this._scale();
+ this.activity.textMsg(_("LEGO Bricks - Phrase Maker with " + this.rowLabels.length + " pitch rows (sorted by frequency, Instrument: " + this.selectedInstrument + ")"));
+ };
+
+ /**
+ * Initializes the audio synthesizer.
+ * @private
+ */
+ this._initAudio = function() {
+ // Create a new synthesizer instance
+ this.synth = new Synth();
+ this.synth.loadSamples();
+
+ // Create the default electronic synth for all pitch playback
+ this.synth.createSynth(0, this.selectedInstrument, this.selectedInstrument, null);
+ };
+
+ /**
+ * Plays a note when a pitch row is clicked.
+ * @private
+ * @param {string} note - The note to play (e.g., "C4")
+ * @param {number} duration - Duration in seconds (default 0.5)
+ */
+ this._playNote = function(note, duration = 0.5) {
+ if (!this.synth) return;
+
+ try {
+ // Play the note using the selected instrument
+ this.synth.trigger(0, note, duration, this.selectedInstrument, null, null, false, 0);
+ } catch (e) {
+ console.error("Error playing note:", e);
+ }
+ };
+
+ /**
+ * Initializes the row headers table with dividing lines.
+ * @private
+ * @returns {void}
+ */
+ this._initializeRowHeaders = function() {
+ this.rowHeaderTable.innerHTML = "";
+ this.rowHeaderTable.style.margin = "0";
+ this.rowHeaderTable.style.padding = "0";
+ this.rowHeaderTable.style.borderSpacing = "0";
+
+ this.matrixData.rows.forEach((rowData, rowIndex) => {
+ const row = this.rowHeaderTable.insertRow();
+ row.style.height = "40px"; // ROW_HEIGHT + "px";
+ row.style.margin = "0";
+ row.style.padding = "0";
+ row.style.position = "relative"; // Needed for absolute positioning of line
+
+ const labelCell = row.insertCell();
+ labelCell.style.display = "flex";
+ labelCell.style.alignItems = "center";
+ labelCell.style.padding = "0 8px";
+ labelCell.style.margin = "0";
+ labelCell.style.fontSize = "13px";
+ labelCell.style.fontWeight = "bold";
+ labelCell.style.border = "none"; // Remove default borders
+ labelCell.style.backgroundColor = rowData.type === "pitch" ? "#77C428" : "#87ceeb";
+ labelCell.style.gap = "8px";
+ labelCell.style.height = "40px"; // ROW_HEIGHT + "px";
+ labelCell.style.lineHeight = "40px"; // ROW_HEIGHT + "px";
+ labelCell.style.boxSizing = "border-box";
+ labelCell.style.cursor = rowData.note ? "pointer" : "default";
+
+ // Add click handler for pitch rows
+ if (rowData.note) {
+ labelCell.onclick = () => {
+ this._playNote(rowData.note);
+ };
+
+ // Visual feedback on click
+ labelCell.onmousedown = () => {
+ labelCell.style.transform = "scale(0.98)";
+ labelCell.style.boxShadow = "inset 0 0 8px rgba(0,0,0,0.2)";
+ };
+
+ labelCell.onmouseup = () => {
+ labelCell.style.transform = "";
+ labelCell.style.boxShadow = "";
+ };
+
+ labelCell.onmouseleave = () => {
+ labelCell.style.transform = "";
+ labelCell.style.boxShadow = "";
+ };
+ }
+
+ // Create icon
+ const icon = document.createElement("div");
+ icon.style.width = "24px";
+ icon.style.height = "24px";
+ icon.style.backgroundColor = "#fff";
+ icon.style.borderRadius = "50%";
+ icon.style.marginRight = "6px";
+ icon.style.flexShrink = "0";
+
+ labelCell.appendChild(icon);
+ labelCell.appendChild(document.createTextNode(rowData.label));
+
+ // Add red line at the bottom of each row (except last)
+ if (rowIndex < this.matrixData.rows.length - 1) {
+ const line = document.createElement("div");
+ line.style.position = "absolute";
+ line.style.left = "0";
+ line.style.right = "0";
+ line.style.bottom = "0";
+ line.style.height = "2px";
+ line.style.backgroundColor = "red";
+ line.style.zIndex = "5";
+ row.appendChild(line);
+ }
+ });
+ };
+
+ /**
+ * Creates the main container with matrix and image canvas.
+ * @returns {void}
+ */
+ this.createMainContainer = function() {
+ const mainContainer = document.createElement("div");
+ mainContainer.style.display = "flex";
+ mainContainer.style.height = "100%";
+ mainContainer.style.overflow = "hidden";
+ mainContainer.style.position = "relative";
+
+ // Create a combined container for matrix and image
+ const contentContainer = document.createElement("div");
+ contentContainer.style.display = "flex";
+ contentContainer.style.flex = "1";
+ contentContainer.style.overflow = "hidden";
+
+ // Create matrix container (just for row headers)
+ const rowHeaderContainer = document.createElement("div");
+ rowHeaderContainer.style.flex = "0 0 auto";
+ rowHeaderContainer.style.backgroundColor = "#cccccc";
+ rowHeaderContainer.style.overflowY = "auto";
+ rowHeaderContainer.style.width = "180px";
+ rowHeaderContainer.style.borderRight = "2px solid #888";
+
+ // Create matrix table (just for row headers)
+ this.rowHeaderTable = document.createElement("table");
+ this.rowHeaderTable.style.borderCollapse = "collapse";
+ this.rowHeaderTable.style.width = "100%";
+
+ rowHeaderContainer.appendChild(this.rowHeaderTable);
+
+ // Create image canvas container
+ const imageCanvas = document.createElement("div");
+ imageCanvas.style.flex = "1";
+ imageCanvas.style.height = "100%";
+ imageCanvas.style.backgroundColor = "#f5f5f5";
+ imageCanvas.style.display = "flex";
+ imageCanvas.style.flexDirection = "column";
+ imageCanvas.style.overflow = "hidden";
+ imageCanvas.style.position = "relative";
+
+ // Create image display area
+ this.imageDisplayArea = document.createElement("div");
+ this.imageDisplayArea.style.position = "relative";
+ this.imageDisplayArea.style.flex = "1";
+ this.imageDisplayArea.style.display = "flex";
+ this.imageDisplayArea.style.alignItems = "center";
+ this.imageDisplayArea.style.justifyContent = "center";
+ this.imageDisplayArea.style.padding = "0"; // Removed padding to prevent misalignment
+ this.imageDisplayArea.style.overflow = "hidden";
+
+ // Create placeholder text
+ this.imagePlaceholder = document.createElement("div");
+ this.imagePlaceholder.style.color = "#888";
+ this.imagePlaceholder.style.fontSize = "16px";
+ this.imagePlaceholder.style.textAlign = "center";
+ this.imagePlaceholder.style.fontStyle = "italic";
+ this.imagePlaceholder.textContent = "Click upload button to add an image";
+
+ this.imageDisplayArea.appendChild(this.imagePlaceholder);
+
+ // Create grid overlay
+ this.gridOverlay = document.createElement("div");
+ this.gridOverlay.style.position = "absolute";
+ this.gridOverlay.style.top = "0";
+ this.gridOverlay.style.left = "0";
+ this.gridOverlay.style.right = "0";
+ this.gridOverlay.style.bottom = "0";
+ this.gridOverlay.style.pointerEvents = "none";
+ this.gridOverlay.style.zIndex = "10";
+
+ imageCanvas.appendChild(this.imageDisplayArea);
+ imageCanvas.appendChild(this.gridOverlay);
+
+ // Create zoom controls (positioned absolutely to prevent layout shifting)
+ this.createZoomControls();
+ imageCanvas.appendChild(this.zoomControls);
+
+ // Add both to content container
+ contentContainer.appendChild(rowHeaderContainer);
+ contentContainer.appendChild(imageCanvas);
+
+ // Add content container to main container
+ mainContainer.appendChild(contentContainer);
+
+ this.widgetWindow.getWidgetBody().appendChild(mainContainer);
+
+ // Create hidden file input
+ this.fileInput = document.createElement("input");
+ this.fileInput.type = "file";
+ this.fileInput.accept = "image/*";
+ this.fileInput.style.display = "none";
+ this.fileInput.onchange = (e) => this._handleImageUpload(e);
+ document.body.appendChild(this.fileInput);
+
+ // Initialize row headers
+ this._initializeRowHeaders();
+ };
+
+ /**
+ * Creates zoom controls with precise adjustments.
+ * @returns {void}
+ */
+ this.createZoomControls = function() {
+ this.zoomControls = document.createElement("div");
+ this.zoomControls.style.display = "none";
+ this.zoomControls.style.position = "absolute"; // Changed to absolute positioning
+ this.zoomControls.style.bottom = "0";
+ this.zoomControls.style.left = "180px"; // Align with image area
+ this.zoomControls.style.right = "0";
+ this.zoomControls.style.padding = "10px";
+ this.zoomControls.style.backgroundColor = "#f0f0f0";
+ this.zoomControls.style.borderTop = "1px solid #888";
+ this.zoomControls.style.display = "flex";
+ this.zoomControls.style.alignItems = "center";
+ this.zoomControls.style.gap = "8px";
+ this.zoomControls.style.zIndex = "20"; // Ensure it's above the grid
+
+ // Instrument selector
+ const instrumentLabel = document.createElement("span");
+ instrumentLabel.textContent = "Instrument:";
+ instrumentLabel.style.fontSize = "12px";
+ instrumentLabel.style.fontWeight = "bold";
+
+ this.instrumentSelect = document.createElement("select");
+ this.instrumentSelect.style.fontSize = "12px";
+ this.instrumentSelect.style.marginRight = "16px";
+
+ // Add instrument options
+ const instruments = [
+ "electronic synth",
+ "piano",
+ "guitar",
+ "acoustic guitar",
+ "electric guitar",
+ "violin",
+ "viola",
+ "cello",
+ "bass",
+ "flute",
+ "clarinet",
+ "saxophone",
+ "trumpet",
+ "trombone",
+ "oboe",
+ "tuba",
+ "banjo",
+ "sine",
+ "square",
+ "sawtooth",
+ "triangle"
+ ];
+
+ instruments.forEach(instrument => {
+ const option = document.createElement("option");
+ option.value = instrument;
+ option.textContent = instrument.charAt(0).toUpperCase() + instrument.slice(1);
+ if (instrument === this.selectedInstrument) {
+ option.selected = true;
+ }
+ this.instrumentSelect.appendChild(option);
+ });
+
+ this.instrumentSelect.onchange = () => this._changeInstrument();
+
+ const zoomLabel = document.createElement("span");
+ zoomLabel.textContent = "Zoom:";
+ zoomLabel.style.fontSize = "12px";
+
+ const zoomOut = document.createElement("button");
+ zoomOut.textContent = "−";
+ zoomOut.onclick = () => this._adjustZoom(-0.01);
+
+ this.zoomSlider = document.createElement("input");
+ this.zoomSlider.type = "range";
+ this.zoomSlider.min = "0.1";
+ this.zoomSlider.max = "3";
+ this.zoomSlider.step = "0.01";
+ this.zoomSlider.value = "1";
+ this.zoomSlider.style.width = "100px";
+ this.zoomSlider.oninput = () => this._handleZoom();
+
+ const zoomIn = document.createElement("button");
+ zoomIn.textContent = "+";
+ zoomIn.onclick = () => this._adjustZoom(0.01);
+
+ this.zoomValue = document.createElement("span");
+ this.zoomValue.textContent = "100%";
+ this.zoomValue.style.fontSize = "12px";
+ this.zoomValue.style.minWidth = "40px";
+
+ // Add separator
+ const separator = document.createElement("span");
+ separator.textContent = "|";
+ separator.style.margin = "0 8px";
+ separator.style.color = "#888";
+
+ // Add vertical spacing controls
+ const spacingLabel = document.createElement("span");
+ spacingLabel.textContent = "Column Spacing:";
+ spacingLabel.style.fontSize = "12px";
+
+ const spacingOut = document.createElement("button");
+ spacingOut.textContent = "−";
+ spacingOut.onclick = () => this._adjustVerticalSpacing(-5);
+
+ this.spacingSlider = document.createElement("input");
+ this.spacingSlider.type = "range";
+ this.spacingSlider.min = "2";
+ this.spacingSlider.max = "200";
+ this.spacingSlider.step = "1";
+ this.spacingSlider.value = "50";
+ this.spacingSlider.style.width = "100px";
+ this.spacingSlider.oninput = () => this._handleVerticalSpacing();
+
+ const spacingIn = document.createElement("button");
+ spacingIn.textContent = "+";
+ spacingIn.onclick = () => this._adjustVerticalSpacing(1);
+
+ this.spacingValue = document.createElement("span");
+ this.spacingValue.textContent = "50px";
+ this.spacingValue.style.fontSize = "12px";
+ this.spacingValue.style.minWidth = "40px";
+
+ this.zoomControls.appendChild(instrumentLabel);
+ this.zoomControls.appendChild(this.instrumentSelect);
+ this.zoomControls.appendChild(zoomLabel);
+ this.zoomControls.appendChild(zoomOut);
+ this.zoomControls.appendChild(this.zoomSlider);
+ this.zoomControls.appendChild(zoomIn);
+ this.zoomControls.appendChild(this.zoomValue);
+ this.zoomControls.appendChild(separator);
+ this.zoomControls.appendChild(spacingLabel);
+ this.zoomControls.appendChild(spacingOut);
+ this.zoomControls.appendChild(this.spacingSlider);
+ this.zoomControls.appendChild(spacingIn);
+ this.zoomControls.appendChild(this.spacingValue);
+ };
+
+ /**
+ * Initializes the matrix table.
+ * @private
+ * @returns {void}
+ */
+ this._initializeMatrix = function() {
+ this.matrixTable.innerHTML = "";
+
+ this.matrixData.rows.forEach((rowData, rowIndex) => {
+ const row = this.matrixTable.insertRow();
+
+ // Row label cell
+ const labelCell = row.insertCell();
+ labelCell.style.width = "180px";
+ labelCell.style.height = ROW_HEIGHT + "px";
+ labelCell.style.display = "flex";
+ labelCell.style.alignItems = "center";
+ labelCell.style.padding = "0 8px";
+ labelCell.style.fontSize = "13px";
+ labelCell.style.fontWeight = "bold";
+ labelCell.style.border = "1px solid #888";
+ labelCell.style.position = "sticky";
+ labelCell.style.left = "0";
+ labelCell.style.zIndex = "10";
+ labelCell.style.gap = "8px";
+ labelCell.style.backgroundColor = rowData.type === "pitch" ? "#77C428" : "#87ceeb";
+
+ // Create icon (placeholder since we don't have actual icons)
+ const icon = document.createElement("div");
+ icon.style.width = "24px";
+ icon.style.height = "24px";
+ icon.style.backgroundColor = "#fff";
+ icon.style.borderRadius = "50%";
+ icon.style.marginRight = "6px";
+
+ labelCell.appendChild(icon);
+ labelCell.appendChild(document.createTextNode(rowData.label));
+
+ });
+ };
+
+ /**
+ * Saves the current phrase as action blocks.
+ * @private
+ * @returns {void}
+ */
+ this._savePhrase = function() {
+ if (!this.colorData || this.colorData.length === 0) {
+ this.activity.textMsg(_("No color data to save. Please scan an image first."));
+ return;
+ }
+
+ // Collect notes to play from color detection data
+ this._collectNotesToPlay();
+
+ if (this._notesToPlay.length === 0) {
+ this.activity.textMsg(_("No notes detected from color scanning."));
+ return;
+ }
+
+ // Hide palettes for updating
+ for (const name in this.activity.blocks.palettes.dict) {
+ this.activity.blocks.palettes.dict[name].hideMenu(true);
+ }
+ this.activity.refreshCanvas();
+
+ // Create action block stack
+ const newStack = [
+ [0, ["action", { collapsed: true }], 100, 100, [null, 1, null, null]],
+ [1, ["text", { value: _("LEGO phrase") }], 0, 0, [0]]
+ ];
+ let endOfStackIdx = 0;
+
+ // Process each note in the sequence
+ for (let i = 0; i < this._notesToPlay.length; i++) {
+ const note = this._notesToPlay[i];
+
+ // Add the Note block and its value
+ const idx = newStack.length;
+ newStack.push([idx, "newnote", 0, 0, [endOfStackIdx, idx + 1, idx + 2, null]]);
+ const n = newStack[idx][4].length;
+
+ if (i === 0) {
+ // Connect to action block
+ newStack[endOfStackIdx][4][n - 2] = idx;
+ } else {
+ // Connect to previous note block
+ newStack[endOfStackIdx][4][n - 1] = idx;
+ }
+
+ endOfStackIdx = idx;
+
+ // Add note duration (note value as fraction)
+ const delta = 5; // We're adding 4 blocks: vspace, divide, number, number
+
+ // Add vspace to prevent divide block from obscuring the pitch block
+ newStack.push([idx + 1, "vspace", 0, 0, [idx, idx + delta]]);
+
+ // Note value saved as a fraction
+ let numerator, denominator;
+ if (note.noteValue === 1.5) {
+ // Dotted half note = 3/2
+ numerator = 3;
+ denominator = 2;
+ } else if (note.noteValue === 0.5) {
+ // Double whole note = 1/2 (very long note)
+ numerator = 1;
+ denominator = 2;
+ } else if (Number.isInteger(note.noteValue)) {
+ // Standard note values (1, 2, 4, 8, etc.)
+ numerator = 1;
+ denominator = note.noteValue;
+ } else {
+ // For other fractional values, convert to proper fraction
+ numerator = 1;
+ denominator = Math.round(1 / note.noteValue);
+ }
+
+ newStack.push([idx + 2, "divide", 0, 0, [idx, idx + 3, idx + 4]]);
+ newStack.push([idx + 3, ["number", { value: numerator }], 0, 0, [idx + 2]]);
+ newStack.push([idx + 4, ["number", { value: denominator }], 0, 0, [idx + 2]]);
+
+ // Connect the Note block flow to the divide and vspace blocks
+ newStack[idx][4][1] = idx + 2; // divide block
+ newStack[idx][4][2] = idx + 1; // vspace block
+
+ let lastConnection = null;
+ let previousBlock = idx + 1; // vspace block
+ let thisBlock = idx + delta;
+
+ if (note.pitches.length === 0 || note.isRest) {
+ // Add rest block
+ newStack.push([thisBlock, "rest2", 0, 0, [previousBlock, lastConnection]]);
+ } else {
+ // Add pitch blocks for each note
+ for (let j = 0; j < note.pitches.length; j++) {
+ const pitch = note.pitches[j];
+
+ // Determine if this is the last pitch block
+ if (j === note.pitches.length - 1) {
+ lastConnection = null;
+ } else {
+ lastConnection = thisBlock + 3;
+ }
+
+ // Add pitch block
+ newStack.push([
+ thisBlock,
+ "pitch",
+ 0,
+ 0,
+ [previousBlock, thisBlock + 1, thisBlock + 2, lastConnection]
+ ]);
+
+ // Add pitch name
+ newStack.push([
+ thisBlock + 1,
+ ["solfege", { value: pitch.solfege }],
+ 0,
+ 0,
+ [thisBlock]
+ ]);
+
+ // Add octave number
+ newStack.push([
+ thisBlock + 2,
+ ["number", { value: pitch.octave }],
+ 0,
+ 0,
+ [thisBlock]
+ ]);
+
+ thisBlock += 3;
+ previousBlock = thisBlock - 3;
+ }
+ }
+ }
+
+ // Load the new blocks
+ this.activity.blocks.loadNewBlocks(newStack);
+ this.activity.textMsg(_("LEGO phrase saved as action blocks with ") + this._notesToPlay.length + _(" notes"));
+ };
+
+ /**
+ * Collects notes to play from color detection data.
+ * @private
+ */
+ this._collectNotesToPlay = function() {
+ this._notesToPlay = [];
+
+ if (!this.colorData || this.colorData.length === 0) {
+ return;
+ }
+
+ // Analyze column boundaries to determine note timing
+ const columnBoundaries = this._analyzeColumnBoundaries();
+
+ // Filter and merge small segments to meet minimum 1/8 note duration
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ // For each time column, collect the notes that should play
+ for (let colIndex = 0; colIndex < filteredBoundaries.length - 1; colIndex++) {
+ const startTime = filteredBoundaries[colIndex];
+ const endTime = filteredBoundaries[colIndex + 1];
+ const duration = endTime - startTime;
+
+ // Calculate note value based on duration - updated mapping per requirements
+ // <350ms ignored completely (handled in filtering)
+ // 350-750ms: 1/8 note, 750-1500ms: 1/4 note, 1500-3000ms: 1/2 note, 3000+ms: full note
+ let noteValue;
+ if (duration < 750) noteValue = 8; // eighth note (350-750ms)
+ else if (duration < 1500) noteValue = 4; // quarter note (750-1500ms)
+ else if (duration < 3000) noteValue = 2; // half note (1500-3000ms)
+ else noteValue = 1; // whole note (3000ms+)
+ let hasNonGreenColor = false;
+
+ // Check each row for non-green colors in this time range
+ this.colorData.forEach((rowData, rowIndex) => {
+ if (rowData.colorSegments) {
+ let currentTime = 0;
+
+ for (const segment of rowData.colorSegments) {
+ const segmentStart = currentTime;
+ const segmentEnd = currentTime + segment.duration;
+
+ // Check if this segment overlaps with our time column
+ if (segmentStart < endTime && segmentEnd > startTime) {
+ // Calculate the actual overlap duration
+ const overlapStart = Math.max(segmentStart, startTime);
+ const overlapEnd = Math.min(segmentEnd, endTime);
+ const overlapDuration = overlapEnd - overlapStart;
+
+ // Only count as significant if overlap is substantial (>350ms)
+ // This prevents spillovers <350ms across blue lines from creating duplicate notes
+ if (overlapDuration > 350) {
+ // Check if color is not green (meaning note should play)
+ if (segment.color !== "green") {
+ hasNonGreenColor = true;
+
+ // Convert row data to pitch information
+ const pitch = this._convertRowToPitch(rowData);
+ if (pitch && !pitches.some(p => p.solfege === pitch.solfege && p.octave === pitch.octave)) {
+ pitches.push(pitch);
+ }
+ }
+ } else if (segment.color !== "green") {
+ // Ignore small overlaps without logging
+ }
+ }
+
+ currentTime += segment.duration;
+ }
+ }
+ });
+
+ // Add note or rest to the sequence
+ this._notesToPlay.push({
+ pitches: pitches,
+ noteValue: noteValue,
+ duration: duration,
+ isRest: !hasNonGreenColor || pitches.length === 0
+ });
+ }
+ };
+
+ /**
+ * Filters out small segments completely (no merging, just elimination).
+ * Updated: <350ms segments are ignored and added to whichever side's blue line is taking majority.
+ * @private
+ * @param {Array} boundaries - Array of time boundaries
+ * @returns {Array} Filtered boundaries with only segments >= 350ms duration
+ */
+ this._filterSmallSegments = function(boundaries) {
+ if (boundaries.length <= 2) return boundaries;
+
+ const minDuration = 350; // Minimum duration for valid segments (350ms)
+ const filteredBoundaries = [boundaries[0]]; // Always keep the start boundary
+
+ // Process each potential segment
+ for (let i = 1; i < boundaries.length; i++) {
+ const segmentDuration = boundaries[i] - filteredBoundaries[filteredBoundaries.length - 1];
+
+ // Only add this boundary if it creates a segment that meets the minimum duration
+ if (segmentDuration >= minDuration) {
+ filteredBoundaries.push(boundaries[i]);
+ }
+ // If segment is too small (<350ms), we skip this boundary entirely
+ // The time gets absorbed into the adjacent larger segment
+ }
+
+ // Ensure we have at least start and end boundaries
+ if (filteredBoundaries.length === 1 && boundaries.length > 1) {
+ // If we filtered out everything, add the final boundary to create one long segment
+ filteredBoundaries.push(boundaries[boundaries.length - 1]);
+ }
+
+ return filteredBoundaries;
+ };
+
+ /**
+ * Analyzes color segments to determine column boundaries.
+ * @private
+ * @returns {Array} Array of time boundaries in milliseconds
+ */
+ this._analyzeColumnBoundaries = function() {
+ const boundaries = new Set([0]); // Start with 0
+
+ // Collect all segment end times
+ this.colorData.forEach(rowData => {
+ if (rowData.colorSegments) {
+ let currentTime = 0;
+ rowData.colorSegments.forEach(segment => {
+ currentTime += segment.duration;
+ boundaries.add(currentTime);
+ });
+ }
+ });
+
+ // Convert to sorted array
+ const sortedBoundaries = Array.from(boundaries).sort((a, b) => a - b);
+
+ // Merge boundaries that are very close together (within 50ms)
+ const mergedBoundaries = [sortedBoundaries[0]];
+ for (let i = 1; i < sortedBoundaries.length; i++) {
+ if (sortedBoundaries[i] - mergedBoundaries[mergedBoundaries.length - 1] > 50) {
+ mergedBoundaries.push(sortedBoundaries[i]);
+ }
+ }
+
+ return mergedBoundaries;
+ };
+
+ /**
+ * Converts row data to pitch information.
+ * @private
+ * @param {Object} rowData - Row data containing note information
+ * @returns {Object} Pitch object with solfege and octave
+ */
+ this._convertRowToPitch = function(rowData) {
+ if (!rowData.note) return null;
+
+ // Parse note string (e.g., "C4", "D5", etc.)
+ const noteMatch = rowData.note.match(/^([A-G][#b]?)(\d+)$/);
+ if (!noteMatch) return null;
+
+ const noteName = noteMatch[1];
+ const octave = parseInt(noteMatch[2]);
+
+ // Convert note name to solfege
+ const noteToSolfege = {
+ "C": "do",
+ "C#": "do♯", "Db": "re♭",
+ "D": "re",
+ "D#": "re♯", "Eb": "mi♭",
+ "E": "mi",
+ "F": "fa",
+ "F#": "fa♯", "Gb": "sol♭",
+ "G": "sol",
+ "G#": "sol♯", "Ab": "la♭",
+ "A": "la",
+ "A#": "la♯", "Bb": "ti♭",
+ "B": "ti"
+ };
+
+ const solfege = noteToSolfege[noteName] || "do";
+
+ return {
+ solfege: solfege,
+ octave: octave
+ };
+ };
+
+ /**
+ * Exports the current phrase.
+ * @private
+ * @returns {void}
+ */
+ this._exportPhrase = function() {
+ const phraseData = {
+ selectedCells: Array.from(this.matrixData.selectedCells),
+ rows: this.matrixData.rows.map(row => ({ type: row.type, label: row.label }))
+ };
+
+ this.activity.textMsg(_("Exporting phrase data: ") + JSON.stringify(phraseData));
+ };
+
+ /**
+ * Clears all selected cells.
+ * @private
+ * @returns {void}
+ */
+ this._clearPhrase = function() {
+ this.matrixData.selectedCells.clear();
+ const selectedCells = this.matrixTable.querySelectorAll("[data-cell-id]");
+ selectedCells.forEach(cell => {
+ cell.style.backgroundColor = "";
+ const dot = cell.querySelector(".cell-dot");
+ if (dot) cell.removeChild(dot);
+ });
+ this.activity.textMsg(_("Phrase cleared"));
+ };
+
+ /**
+ * Uploads an image.
+ * @private
+ * @returns {void}
+ */
+ this._uploadImage = function() {
+ this.fileInput.click();
+ };
+
+ /**
+ * Handles image upload.
+ * @private
+ * @param {Event} event - The file input change event.
+ * @returns {void}
+ */
+ this._handleImageUpload = function(event) {
+ const file = event.target.files[0];
+ if (file && file.type.startsWith("image/")) {
+ const reader = new FileReader();
+ reader.onload = (e) => {
+ this.imageDisplayArea.innerHTML = "";
+
+ this.imageWrapper = document.createElement("div");
+ this.imageWrapper.style.position = "absolute";
+ this.imageWrapper.style.left = "0px";
+ this.imageWrapper.style.top = "0px";
+ this.imageWrapper.style.transformOrigin = "top left";
+ this.imageWrapper.style.cursor = "grab";
+
+ const img = document.createElement("img");
+ img.src = e.target.result;
+ img.style.maxWidth = "100%";
+ img.style.maxHeight = "100%";
+ img.style.objectFit = "contain";
+ img.style.borderRadius = "8px";
+ img.style.boxShadow = "0 2px 8px rgba(0,0,0,0.2)";
+
+ this.imageWrapper.appendChild(img);
+ this.imageDisplayArea.appendChild(this.imageWrapper);
+
+ this._makeImageDraggable(this.imageWrapper);
+ this._showZoomControls();
+ this._drawGridLines();
+
+ this.activity.textMsg(_("Image uploaded successfully"));
+ };
+ reader.readAsDataURL(file);
+ }
+ };
+
+ /**
+ * Starts webcam.
+ * @private
+ * @returns {void}
+ */
+ this._startWebcam = function() {
+ this.imageDisplayArea.innerHTML = "";
+
+ this.imageWrapper = document.createElement("div");
+ this.imageWrapper.style.position = "absolute";
+ this.imageWrapper.style.left = "0px";
+ this.imageWrapper.style.top = "0px";
+ this.imageWrapper.style.transformOrigin = "top left";
+ this.imageWrapper.style.cursor = "grab";
+
+ this.webcamVideo = document.createElement("video");
+ this.webcamVideo.autoplay = true;
+ this.webcamVideo.playsInline = true;
+ this.webcamVideo.style.maxWidth = "100%";
+ this.webcamVideo.style.maxHeight = "100%";
+
+ this.imageWrapper.appendChild(this.webcamVideo);
+ this.imageDisplayArea.appendChild(this.imageWrapper);
+
+ navigator.mediaDevices.getUserMedia({ video: true })
+ .then((stream) => {
+ this.webcamVideo.srcObject = stream;
+ this._makeImageDraggable(this.imageWrapper);
+ this._showZoomControls();
+ this._drawGridLines();
+ this.activity.textMsg(_("Webcam started"));
+ })
+ .catch((err) => {
+ this.activity.textMsg(_("Webcam access denied: ") + err.message);
+ });
+ };
+
+ /**
+ * Stops webcam.
+ * @private
+ * @returns {void}
+ */
+ this._stopWebcam = function() {
+ if (this.webcamVideo && this.webcamVideo.srcObject) {
+ const tracks = this.webcamVideo.srcObject.getTracks();
+ tracks.forEach(track => track.stop());
+ }
+ };
+
+ /**
+ * Makes image draggable.
+ * @private
+ * @param {HTMLElement} wrapper - The image wrapper element.
+ * @returns {void}
+ */
+ this._makeImageDraggable = function(wrapper) {
+ let isDragging = false;
+ let startX, startY, initialX, initialY;
+
+ // Set fixed dimensions for the wrapper
+ wrapper.style.width = "100%";
+ wrapper.style.height = "100%";
+ wrapper.style.overflow = "hidden";
+
+ wrapper.onmousedown = (e) => {
+ isDragging = true;
+ startX = e.clientX;
+ startY = e.clientY;
+ initialX = parseFloat(wrapper.style.left) || 0;
+ initialY = parseFloat(wrapper.style.top) || 0;
+ wrapper.style.cursor = "grabbing";
+ e.preventDefault();
+ };
+
+ document.onmousemove = (e) => {
+ if (!isDragging) return;
+ const dx = e.clientX - startX;
+ const dy = e.clientY - startY;
+ wrapper.style.left = `${initialX + dx}px`;
+ wrapper.style.top = `${initialY + dy}px`;
+ };
+
+ document.onmouseup = () => {
+ if (isDragging) {
+ isDragging = false;
+ wrapper.style.cursor = "grab";
+ }
+ };
+ };
+
+ /**
+ * Converts RGB values to a named color category.
+ * @private
+ */
+ this._getColorFamily = function(r, g, b) {
+ const hsl = this._rgbToHsl(r, g, b);
+ const [hue, saturation, lightness] = hsl;
+
+ let name = "unknown";
+ if (lightness < 20) name = "black";
+ else if (lightness > 80 && saturation < 20) name = "white";
+ else if (saturation < 25) name = "gray";
+ else if (hue >= 0 && hue < 30) name = "red";
+ else if (hue >= 30 && hue < 60) name = "orange";
+ else if (hue >= 60 && hue < 90) name = "yellow";
+ else if (hue >= 90 && hue < 150) name = "green"; // FIXED: Added green detection!
+ else if (hue >= 150 && hue < 210) name = "cyan";
+ else if (hue >= 210 && hue < 270) name = "blue";
+ else if (hue >= 270 && hue < 330) name = "magenta";
+ else if (hue >= 330 && hue <= 360) name = "red";
+
+ return {
+ name: name,
+ hue: hue,
+ saturation: saturation,
+ lightness: lightness
+ };
+ };
+
+ /**
+ * Gets color family from HSL values
+ * @private
+ * @param {number} h - Hue (0-360)
+ * @param {number} s - Saturation (0-100)
+ * @param {number} l - Lightness (0-100)
+ * @returns {object} Color family object
+ */
+ this._getColorFamily = function(h, s, l) {
+ // Handle grayscale first
+ if (s < 15) { // Low saturation = grayscale
+ if (l > 85) return { name: "white", hue: 0 };
+ if (l < 15) return { name: "black", hue: 0 };
+ return { name: "gray", hue: 0 };
+ }
+
+ // For saturated colors, determine by hue
+ if (h >= 345 || h < 15) return { name: "red", hue: 0 };
+ if (h >= 15 && h < 45) return { name: "orange", hue: 30 };
+ if (h >= 45 && h < 75) return { name: "yellow", hue: 60 };
+ if (h >= 75 && h < 165) return { name: "green", hue: 120 }; // FIXED: Proper green range!
+ if (h >= 165 && h < 195) return { name: "cyan", hue: 180 };
+ if (h >= 195 && h < 255) return { name: "blue", hue: 240 };
+ if (h >= 255 && h < 285) return { name: "purple", hue: 270 };
+ if (h >= 285 && h < 315) return { name: "magenta", hue: 300 };
+ if (h >= 315 && h < 345) return { name: "pink", hue: 330 };
+
+ return { name: "unknown", hue: h };
+ };
+
+ /**
+ * Gets color family by name
+ * @private
+ * @param {string} colorName - The color name
+ * @returns {object} Color family object
+ */
+ this._getColorFamilyByName = function(colorName) {
+ const colorFamilies = {
+ "red": { name: "red", hue: 0 },
+ "orange": { name: "orange", hue: 30 },
+ "yellow": { name: "yellow", hue: 60 },
+ "green": { name: "green", hue: 120 }, // FIXED: Added green!
+ "cyan": { name: "cyan", hue: 180 }, // FIXED: Added cyan!
+ "blue": { name: "blue", hue: 240 },
+ "purple": { name: "purple", hue: 270 },
+ "magenta": { name: "magenta", hue: 300 }, // FIXED: Added magenta!
+ "pink": { name: "pink", hue: 330 },
+ "white": { name: "white", hue: 0 }, // FIXED: Added white!
+ "black": { name: "black", hue: 0 }, // FIXED: Added black!
+ "gray": { name: "gray", hue: 0 } // FIXED: Added gray!
+ };
+ return colorFamilies[colorName] || null;
+ };
+
+ /**
+ * Converts RGB to HSL.
+ * @private
+ */
+ this._rgbToHsl = function(r, g, b) {
+ r /= 255;
+ g /= 255;
+ b /= 255;
+
+ const max = Math.max(r, g, b), min = Math.min(r, g, b);
+ let h = 0, s = 0, l = (max + min) / 2;
+
+ if (max !== min) {
+ const d = max - min;
+ s = l > 0.5 ? d / (2 - max - min) : d / (max + min);
+
+ switch (max) {
+ case r: h = (g - b) / d + (g < b ? 6 : 0); break;
+ case g: h = (b - r) / d + 2; break;
+ case b: h = (r - g) / d + 4; break;
+ }
+
+ h *= 60;
+ }
+
+ return [Math.round(h), Math.round(s * 100), Math.round(l * 100)];
+ };
+
+
+ /**
+ * Checks if two colors are similar enough to not count as a change.
+ * @private
+ * @param {object} color1 - First color family object.
+ * @param {object} color2 - Second color family object.
+ * @returns {boolean} True if colors are similar.
+ */
+ this._colorsAreSimilar = function(color1, color2) {
+ if (!color1 || !color2) return false;
+
+ // Same color family
+ if (color1.name === color2.name) return true;
+
+ // Black/white/gray similarities
+ if ((color1.name === "black" && color2.name === "gray" && color2.lightness < 60) ||
+ (color2.name === "black" && color1.name === "gray" && color1.lightness < 60)) {
+ return true;
+ }
+
+ return false;
+ };
+
+ /**
+ * Checks if a canvas row is within the actual image bounds and returns appropriate color
+ * @private
+ * @param {object} line - The scanning line object
+ * @param {HTMLElement} mediaElement - The image or video element
+ * @returns {object|null} Color family object or null if should continue with normal sampling
+ */
+ this._getColorForCanvasRow = function(line, mediaElement) {
+ // Get the actual image display area
+ const imageRect = mediaElement.getBoundingClientRect();
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+
+ // Calculate if this row is within the image bounds
+ const rowTopInOverlay = line.topPos;
+ const rowBottomInOverlay = line.bottomPos;
+
+ // Get image position relative to overlay (account for positioning/dragging)
+ const imageTop = imageRect.top - overlayRect.top;
+ const imageBottom = imageTop + imageRect.height;
+
+ // Check if row is completely outside image bounds (underflow/overflow)
+ if (rowBottomInOverlay <= imageTop || rowTopInOverlay >= imageBottom) {
+ // Row is outside image - return green for empty canvas areas
+ return this._getColorFamilyByName("green");
+ }
+
+ // Row is within or partially within image bounds - continue with normal color sampling
+ return null;
+ };
+
+
+ /**
+ * Shows zoom controls.
+ * @private
+ * @returns {void}
+ */
+ this._showZoomControls = function() {
+ this.zoomControls.style.display = "flex";
+ this.currentZoom = 1;
+ this.zoomSlider.value = "1";
+ this.zoomValue.textContent = "100%";
+
+ // Initialize vertical spacing controls
+ this.spacingSlider.value = this.verticalSpacing.toString();
+ this.spacingValue.textContent = this.verticalSpacing + "px";
+
+ // No need to redraw lines here since zoom controls are absolutely positioned
+ };
+
+ /**
+ * Adjusts zoom level.
+ * @private
+ * @param {number} delta - The zoom adjustment.
+ * @returns {void}
+ */
+ this._adjustZoom = function(delta) {
+ let newVal = parseFloat(this.zoomSlider.value) + delta;
+ newVal = Math.max(0.1, Math.min(3, newVal));
+ this.zoomSlider.value = newVal.toFixed(2);
+ this._handleZoom();
+ };
+
+ /**
+ * Adjusts vertical spacing level.
+ * @private
+ * @param {number} delta - The spacing adjustment.
+ * @returns {void}
+ */
+ this._adjustVerticalSpacing = function(delta) {
+ let newVal = parseFloat(this.spacingSlider.value) + delta;
+ newVal = Math.max(2, Math.min(200, newVal));
+ this.spacingSlider.value = newVal.toString();
+ this._handleVerticalSpacing();
+ };
+
+ /**
+ * Handles vertical spacing changes.
+ * @private
+ * @returns {void}
+ */
+ this._handleVerticalSpacing = function() {
+ this.verticalSpacing = parseFloat(this.spacingSlider.value);
+ this.spacingValue.textContent = this.verticalSpacing + "px";
+
+ setTimeout(() => this._drawGridLines(), 50);
+ };
+
+ /**
+ * Changes the selected instrument for playback.
+ * @private
+ * @returns {void}
+ */
+ this._changeInstrument = function() {
+ this.selectedInstrument = this.instrumentSelect.value;
+
+ // Recreate the synth with the new instrument
+ if (this.synth) {
+ this.synth.createSynth(0, this.selectedInstrument, this.selectedInstrument, null);
+ }
+
+ // Show a message indicating the instrument change
+ this.activity.textMsg(_("Instrument changed to: ") + this.selectedInstrument);
+ };
+
+
+ /**
+ * Handles zoom changes.
+ * @private
+ * @returns {void}
+ */
+ this._handleZoom = function() {
+ if (this.imageWrapper) {
+ this.currentZoom = parseFloat(this.zoomSlider.value);
+ this.imageWrapper.style.transform = `scale(${this.currentZoom})`;
+ this.zoomValue.textContent = Math.round(this.currentZoom * 100) + "%";
+
+ // Ensure the image wrapper maintains its dimensions
+ this.imageWrapper.style.width = "100%";
+ this.imageWrapper.style.height = "100%";
+
+ setTimeout(() => this._drawGridLines(), 50);
+ }
+ };
+
+ /**
+ * Draws grid lines over the image.
+ * @private
+ * @returns {void}
+ */
+ this._drawGridLines = function() {
+ if (!this.rowHeaderTable.rows.length || !this.gridOverlay) return;
+
+ this.gridOverlay.innerHTML = "";
+
+ const numRows = this.matrixData.rows.length;
+
+ // Draw horizontal lines at each row boundary
+ for (let i = 0; i < numRows; i++) {
+ const line = document.createElement("div");
+ line.style.position = "absolute";
+ line.style.left = "0px";
+ line.style.right = "0px";
+ line.style.height = "2px";
+ line.style.backgroundColor = "red";
+ line.style.zIndex = "5";
+
+ // Position line at the bottom of each row
+ const position = (i + 1) * ROW_HEIGHT;
+ line.style.top = `${position}px`;
+
+ this.gridOverlay.appendChild(line);
+ }
+
+ // Draw vertical lines based on spacing
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+ const overlayWidth = overlayRect.width || this.gridOverlay.offsetWidth || 800; // fallback width
+
+ if (overlayWidth > 0) {
+ const numVerticalLines = Math.floor(overlayWidth / this.verticalSpacing);
+
+ for (let i = 1; i <= numVerticalLines; i++) {
+ const vline = document.createElement("div");
+ vline.style.position = "absolute";
+ vline.style.top = "0px";
+ vline.style.bottom = "0px";
+ vline.style.width = "2px";
+ vline.style.backgroundColor = "blue";
+ vline.style.zIndex = "5";
+
+ // Position vertical line
+ const position = i * this.verticalSpacing;
+ vline.style.left = `${position}px`;
+
+ this.gridOverlay.appendChild(vline);
+ }
+ }
+ };
+
+ /**
+ * Scales the widget window and canvas based on the window's state.
+ * @private
+ * @returns {void}
+ */
+ this._scale = function() {
+ // Redraw grid lines after scaling
+ setTimeout(() => this._drawGridLines(), 300);
+ };
+
+ /**
+ * Shows the widget.
+ * @returns {void}
+ */
+ this.show = function() {
+ if (this.widgetWindow) {
+ this.widgetWindow.show();
+ }
+ };
+
+ /**
+ * Updates widget parameters.
+ * @param {object} params - The parameters to update.
+ * @returns {void}
+ */
+ this.updateParams = function(params) {
+ // Handle parameter updates if needed
+ if (params.zoom && this.zoomSlider) {
+ this.zoomSlider.value = params.zoom;
+ this._handleZoom();
+ }
+ };
+
+ /**
+ * Plays the current musical phrase with vertical scanning lines.
+ * @private
+ */
+ this._playPhrase = function() {
+ // Clear any existing animation
+ this._stopPlayback();
+ this.activity.textMsg(_("Scanning image with vertical lines..."));
+
+ // Get all grid lines (sorted by position)
+ const gridLines = Array.from(this.gridOverlay.querySelectorAll("div"))
+ .filter(el => el.style.backgroundColor === "red")
+ .sort((a, b) => {
+ const aTop = parseFloat(a.style.top);
+ const bTop = parseFloat(b.style.top);
+ return aTop - bTop;
+ });
+
+ // Create scanning lines for each musical note row
+ this.scanningLines = [];
+ this.colorData = [];
+
+ // Get the actual canvas/overlay dimensions
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+ const canvasHeight = overlayRect.height || (this.matrixData.rows.length * ROW_HEIGHT);
+ const totalNoteRows = this.matrixData.rows.filter(row => row.note).length;
+
+ // Create entries and scanning lines for each musical note
+ this.matrixData.rows.forEach((row, index) => {
+ if (!row.note) return; // Skip non-note rows
+
+ this.colorData.push({
+ note: row.note,
+ label: row.label,
+ colorSegments: []
+ });
+
+ // Calculate vertical position for this note - fixed to canvas grid
+ const topPos = index * ROW_HEIGHT;
+ const bottomPos = (index + 1) * ROW_HEIGHT;
+
+ // Ensure we don't go beyond canvas boundaries
+ const clampedTopPos = Math.max(0, Math.min(topPos, canvasHeight));
+ const clampedBottomPos = Math.max(0, Math.min(bottomPos, canvasHeight));
+
+ // Skip if this row is completely outside canvas bounds
+ if (clampedTopPos >= canvasHeight || clampedBottomPos <= 0) {
+ // Fill this row with green color for the entire duration
+ this.colorData[this.colorData.length - 1].colorSegments.push({
+ color: "green",
+ duration: 5000, // Default scan duration
+ timestamp: performance.now()
+ });
+ return;
+ }
+
+ // Create vertical scanning line
+ const line = document.createElement("div");
+ line.style.position = "absolute";
+ line.style.width = "3px"; // Slightly thicker line for better visibility
+ line.style.height = (clampedBottomPos - clampedTopPos) + "px";
+ line.style.backgroundColor = "rgba(255, 0, 0, 0.7)";
+ line.style.zIndex = "20";
+ line.style.left = "0px";
+ line.style.top = clampedTopPos + "px";
+ line.dataset.lineId = index;
+ this.gridOverlay.appendChild(line);
+
+ this.scanningLines.push({
+ element: line,
+ topPos: clampedTopPos,
+ bottomPos: clampedBottomPos,
+ currentX: 0,
+ currentColor: null,
+ colorStartTime: null,
+ lastSignificantColor: null,
+ completed: false,
+ rowIndex: index
+ });
+ });
+
+ // Animation variables
+ this.isPlaying = true;
+ this.startTime = performance.now();
+ this.lastFrameTime = this.startTime;
+
+ // Start animation
+ this._animateLines();
+ };
+
+ /**
+ * Animates all scanning lines with improved color detection
+ * @private
+ */
+ this._animateLines = function() {
+ if (!this.isPlaying) return;
+
+ const now = performance.now();
+ const deltaTime = (now - this.lastFrameTime) / 1000;
+ this.lastFrameTime = now;
+
+ const containerRect = this.gridOverlay.getBoundingClientRect();
+ // Keep consistent time between vertical blue lines (column spacing)
+ // Target: 500ms between each vertical line for instructor predictability
+ const timeBetweenColumns = 0.5; // seconds per column spacing
+ const scanSpeed = this.verticalSpacing / timeBetweenColumns; // pixels per second
+
+ let allLinesCompleted = true;
+
+ this.scanningLines.forEach(line => {
+ if (line.completed) return;
+
+ // Update horizontal position
+ line.currentX += scanSpeed * deltaTime;
+ const maxX = containerRect.width;
+
+ // Check if we've reached the container boundary
+ if (line.currentX > maxX) {
+ line.completed = true;
+ // Record final color segment if it existed
+ if (line.lastSignificantColor && line.colorStartTime) {
+ const duration = (now - line.colorStartTime) / 1000;
+ this.colorData[line.rowIndex].colorSegments.push({
+ color: line.lastSignificantColor.name,
+ duration: duration,
+ endTime: now - this.startTime
+ });
+ }
+ return;
+ }
+
+ // Check if we've reached the right edge of the actual image
+ if (this._isLineBeyondImageHorizontally(line)) {
+ line.completed = true;
+ // Record final color segment if it existed
+ if (line.lastSignificantColor && line.colorStartTime) {
+ const duration = (now - line.colorStartTime) / 1000;
+ this.colorData[line.rowIndex].colorSegments.push({
+ color: line.lastSignificantColor.name,
+ duration: duration,
+ endTime: now - this.startTime
+ });
+ }
+ return;
+ }
+
+ allLinesCompleted = false;
+
+ // Update line position
+ line.element.style.left = line.currentX + "px";
+
+ // Sample colors across the entire vertical line
+ this._sampleAndDetectColor(line, now);
+ });
+
+ if (allLinesCompleted) {
+ this._stopPlayback();
+ } else {
+ requestAnimationFrame(() => this._animateLines());
+ }
+ };
+
+ /**
+ * Checks if the scanning line has moved beyond the right edge of the actual image
+ * @private
+ * @param {object} line - The scanning line object
+ * @returns {boolean} True if line is beyond image's right edge
+ */
+ this._isLineBeyondImageHorizontally = function(line) {
+ // Get the image or video element
+ let mediaElement = null;
+ if (this.imageWrapper) {
+ mediaElement = this.imageWrapper.querySelector("img") || this.imageWrapper.querySelector("video");
+ }
+
+ if (!mediaElement) {
+ return false; // If no image, let it continue scanning the container
+ }
+
+ // Get the actual image display area
+ const imageRect = mediaElement.getBoundingClientRect();
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+
+ // Calculate image position relative to overlay (account for positioning/dragging)
+ const imageLeft = imageRect.left - overlayRect.left;
+ const imageRight = imageLeft + imageRect.width;
+
+ // Check if the scanning line is beyond the right edge of the image
+ const lineX = line.currentX;
+
+ if (lineX >= imageRight) {
+ return true;
+ }
+
+ return false;
+ };
+
+ /**
+ * Stops the current playback animation.
+ * @private
+ */
+ this._stopPlayback = function() {
+ this.isPlaying = false;
+
+ // Save final color segments for all lines
+ if (this.scanningLines) {
+ const now = performance.now();
+ this.scanningLines.forEach(line => {
+ // Save the final color segment if it exists
+ if (line.currentColor && line.colorStartTime) {
+ const duration = now - line.colorStartTime;
+ if (duration > 350) { // Save final segment if long enough (updated to match new minimum)
+ this._addColorSegment(line.rowIndex, line.currentColor, duration);
+ }
+ }
+
+ // Remove the scanning line element
+ if (line.element && line.element.parentNode) {
+ line.element.parentNode.removeChild(line.element);
+ }
+ });
+ this.scanningLines = null;
+ }
+
+ // Generate color visualization PNG
+ setTimeout(() => {
+ this._generateColorVisualization();
+ this._drawColumnLinesOnCanvas(); // Draw column lines on the overlay
+ }, 100); // Small delay to ensure all data is processed
+ };
+
+
+ /**
+ * Samples and detects colors along a vertical line
+ * @private
+ * @param {object} line - The scanning line object
+ * @param {number} now - Current timestamp
+ */
+ this._sampleAndDetectColor = function(line, now) {
+ // Get the image or video element
+ let mediaElement = null;
+ if (this.imageWrapper) {
+ mediaElement = this.imageWrapper.querySelector("img") || this.imageWrapper.querySelector("video");
+ }
+
+ if (!mediaElement) {
+ console.warn("No image or video element found for color sampling");
+ return;
+ }
+
+ // Check if this row is outside image bounds - if so, use green
+ const boundColor = this._getColorForCanvasRow(line, mediaElement);
+ if (boundColor) {
+ // Row is outside image bounds, force green color
+ if (!line.currentColor || line.currentColor.name !== boundColor.name) {
+ const timeSinceLastChange = line.lastColorChangeTime ? (now - line.lastColorChangeTime) : 1000;
+ if (timeSinceLastChange > 150) {
+ if (line.currentColor && line.colorStartTime) {
+ const duration = now - line.colorStartTime;
+ if (duration > 350) { // Updated to match new minimum note duration
+ this._addColorSegment(line.rowIndex, line.currentColor, duration);
+ }
+ }
+ line.currentColor = boundColor;
+ line.colorStartTime = now;
+ line.lastColorChangeTime = now;
+ }
+ }
+ return; // Don't sample pixels, just use the bound color
+ }
+
+ // Create a temporary canvas to sample pixel data
+ const tempCanvas = document.createElement("canvas");
+ const ctx = tempCanvas.getContext("2d");
+
+ // Set canvas size to match the media element's display size
+ const mediaRect = mediaElement.getBoundingClientRect();
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+
+ tempCanvas.width = mediaElement.naturalWidth || mediaElement.videoWidth || mediaRect.width;
+ tempCanvas.height = mediaElement.naturalHeight || mediaElement.videoHeight || mediaRect.height;
+
+ // Draw the media element to the canvas
+ try {
+ ctx.drawImage(mediaElement, 0, 0, tempCanvas.width, tempCanvas.height);
+ } catch (e) {
+ console.error("Error drawing image to canvas:", e);
+ return;
+ }
+
+ // Get overlay and image positioning
+ const imageRect = mediaElement.getBoundingClientRect();
+
+ // Calculate image position relative to overlay
+ const imageOffsetX = imageRect.left - overlayRect.left;
+ const imageOffsetY = imageRect.top - overlayRect.top;
+
+ // Convert overlay coordinates to image coordinates
+ const overlayX = line.currentX;
+ const overlayY1 = line.topPos;
+ const overlayY2 = line.bottomPos;
+
+ // Check if sampling area is within image bounds horizontally
+ const imageX = overlayX - imageOffsetX;
+ const imageY1 = overlayY1 - imageOffsetY;
+ const imageY2 = overlayY2 - imageOffsetY;
+
+ // Early return if we're outside the horizontal image bounds
+ // (The animation loop will handle stopping the line when it reaches the edge)
+ if (imageX < 0 || imageX >= imageRect.width) {
+ // We're outside the image horizontally - no need to sample
+ return;
+ }
+
+ // Calculate canvas coordinates with proper scaling
+ const scaleX = tempCanvas.width / imageRect.width;
+ const scaleY = tempCanvas.height / imageRect.height;
+
+ const canvasX = Math.floor(imageX * scaleX);
+ const canvasY1 = Math.max(0, Math.floor(imageY1 * scaleY));
+ const canvasY2 = Math.min(tempCanvas.height, Math.floor(imageY2 * scaleY));
+
+ // Sample multiple points along the vertical line
+ const samplePoints = 32;
+ const colorCounts = {};
+ let totalSamples = 0;
+
+ for (let i = 0; i < samplePoints; i++) {
+ const y = canvasY1 + (i * (canvasY2 - canvasY1) / (samplePoints - 1));
+
+ if (canvasX >= 0 && canvasX < tempCanvas.width && y >= 0 && y < tempCanvas.height) {
+ try {
+ const imageData = ctx.getImageData(canvasX, y, 1, 1);
+ const [r, g, b] = imageData.data;
+
+ // Skip very transparent pixels
+ if (imageData.data[3] < 128) continue;
+
+ // Convert RGB to HSL and get color family
+ const [h, s, l] = this._rgbToHsl(r, g, b);
+ const colorFamily = this._getColorFamily(h, s, l);
+
+ // FIXED: Include ALL colors - don't filter out ANY colors!
+ if (colorFamily && colorFamily.name !== "unknown") {
+ colorCounts[colorFamily.name] = (colorCounts[colorFamily.name] || 0) + 1;
+ totalSamples++;
+ }
+ } catch (e) {
+ console.error("Error sampling pixel data:", e);
+ }
+ }
+ }
+
+ // Only proceed if we have enough color samples
+ if (totalSamples < 1) return;
+
+ // Find dominant color (must be at least 25% of samples)
+ let dominantColor = null;
+ let maxCount = 0;
+ const minThreshold = Math.max(1, Math.floor(totalSamples * 0.25));
+
+ for (const [colorName, count] of Object.entries(colorCounts)) {
+ if (count > maxCount && count >= minThreshold) {
+ maxCount = count;
+ dominantColor = this._getColorFamilyByName(colorName);
+ }
+ }
+
+ // Only record significant color changes
+ if (dominantColor && (!line.currentColor || !this._colorsAreSimilar(line.currentColor, dominantColor))) {
+ // Require a minimum time gap between color changes (reduces noise)
+ const timeSinceLastChange = line.lastColorChangeTime ? (now - line.lastColorChangeTime) : 1000;
+
+ if (timeSinceLastChange > 150) {
+ // Color changed - save previous segment if it existed
+ if (line.currentColor && line.colorStartTime) {
+ const duration = now - line.colorStartTime;
+ if (duration > 350) { // Updated to match new minimum note duration
+ this._addColorSegment(line.rowIndex, line.currentColor, duration);
+ }
+ }
+
+ // Start new color segment
+ line.currentColor = dominantColor;
+ line.colorStartTime = now;
+ line.lastColorChangeTime = now;
+ }
+ }
+ };
+
+ /**
+ * Gets color family by name
+ * @private
+ * @param {string} colorName - The color name
+ * @returns {object} Color family object
+ */
+ this._getColorFamilyByName = function(colorName) {
+ const colorFamilies = {
+ "red": { name: "red", hue: 0 },
+ "orange": { name: "orange", hue: 30 },
+ "yellow": { name: "yellow", hue: 60 },
+ "green": { name: "green", hue: 120 }, // FIXED: Added green!
+ "cyan": { name: "cyan", hue: 180 }, // FIXED: Added cyan!
+ "blue": { name: "blue", hue: 240 },
+ "purple": { name: "purple", hue: 270 },
+ "magenta": { name: "magenta", hue: 300 }, // FIXED: Added magenta!
+ "pink": { name: "pink", hue: 330 },
+ "white": { name: "white", hue: 0 }, // FIXED: Added white!
+ "black": { name: "black", hue: 0 }, // FIXED: Added black!
+ "gray": { name: "gray", hue: 0 } // FIXED: Added gray!
+ };
+ return colorFamilies[colorName] || null;
+ };
+
+ /**
+ * Adds a color segment to the data
+ * @private
+ * @param {number} rowIndex - Row index
+ * @param {object} color - Color object
+ * @param {number} duration - Duration in milliseconds
+ */
+ this._addColorSegment = function(rowIndex, color, duration) {
+ if (!this.colorData[rowIndex]) {
+ this.colorData[rowIndex] = {
+ note: this.this.matrixData.rows[rowIndex].note,
+ label: this.this.matrixData.rows[rowIndex].label,
+ colorSegments: []
+ };
+ }
+
+ this.colorData[rowIndex].colorSegments.push({
+ color: color.name,
+ duration: duration,
+ timestamp: performance.now()
+ });
+ };
+
+ /**
+ * Draws column edge lines on the overlay canvas during scanning
+ * @private
+ */
+ this._drawColumnLinesOnCanvas = function() {
+ if (!this.colorData || this.colorData.length === 0) return;
+
+ // Find existing column lines to avoid duplicates
+ const existingColumnLines = this.gridOverlay.querySelectorAll(".column-line");
+ existingColumnLines.forEach(line => line.remove());
+
+ // Get filtered column boundaries (same as used for note export)
+ const columnBoundaries = this._analyzeColumnBoundaries();
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ const overlayRect = this.gridOverlay.getBoundingClientRect();
+ const availableWidth = overlayRect.width || 800;
+
+ // Calculate total duration for proportional positioning
+ const totalDuration = filteredBoundaries[filteredBoundaries.length - 1] - filteredBoundaries[0];
+
+ // Draw blue vertical lines at filtered boundary positions
+ filteredBoundaries.forEach((boundaryTime, index) => {
+ if (index === 0) return; // Skip the first boundary (start)
+
+ // Calculate position based on time proportion
+ const timeFromStart = boundaryTime - filteredBoundaries[0];
+ const x = Math.round((timeFromStart / totalDuration) * availableWidth);
+
+ if (x > 0 && x < availableWidth) {
+ const vline = document.createElement("div");
+ vline.className = "column-line";
+ vline.style.position = "absolute";
+ vline.style.top = "0px";
+ vline.style.bottom = "0px";
+ vline.style.width = "2px";
+ vline.style.backgroundColor = "#0066FF";
+ vline.style.zIndex = "15"; // Above grid lines but below scanning lines
+ vline.style.left = `${x}px`;
+
+ this.gridOverlay.appendChild(vline);
+ }
+ });
+ };
+
+ /**
+ * Detects column edges across all rows and draws vertical lines on the canvas
+ * @private
+ * @param {CanvasRenderingContext2D} ctx - Canvas context
+ * @param {number} canvasWidth - Width of the canvas
+ * @param {number} canvasHeight - Height of the canvas
+ * @param {number} startX - X position where segments start (after labels)
+ * @param {number} availableWidth - Available width for segments
+ */
+ this._drawColumnLines = function(ctx, canvasWidth, canvasHeight, startX, availableWidth) {
+ // Get filtered column boundaries (same as used for note export and overlay)
+ const columnBoundaries = this._analyzeColumnBoundaries();
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ // Calculate total duration for proportional positioning
+ const totalDuration = filteredBoundaries[filteredBoundaries.length - 1] - filteredBoundaries[0];
+
+ // Draw blue vertical lines at filtered boundary positions
+ ctx.strokeStyle = "#0066FF";
+ ctx.lineWidth = 3; // Slightly thicker for PNG visibility
+
+ filteredBoundaries.forEach((boundaryTime, index) => {
+ if (index === 0) return; // Skip the first boundary (start)
+
+ // Calculate X position based on time proportion
+ const timeFromStart = boundaryTime - filteredBoundaries[0];
+ const x = startX + Math.round((timeFromStart / totalDuration) * availableWidth);
+
+ if (x >= startX && x <= startX + availableWidth) {
+ ctx.beginPath();
+ ctx.moveTo(x, 0);
+ ctx.lineTo(x, canvasHeight);
+ ctx.stroke();
+ }
+ });
+ };
+
+ /**
+ * Generates a PNG image visualization of detected color data (for testing)
+ * @private
+ */
+ this._generateColorVisualization = function() {
+ if (!this.colorData || this.colorData.length === 0) {
+ return;
+ }
+
+ // Canvas dimensions
+ const canvasWidth = 800;
+ const rowHeight = 50;
+ const canvasHeight = this.colorData.length * rowHeight;
+
+ // Create canvas
+ const canvas = document.createElement("canvas");
+ canvas.width = canvasWidth;
+ canvas.height = canvasHeight;
+ const ctx = canvas.getContext("2d");
+
+ // Fill background
+ ctx.fillStyle = "#f0f0f0";
+ ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+
+ // Color mapping
+ const colorMap = {
+ "red": "#FF0000",
+ "orange": "#FFA500",
+ "yellow": "#FFFF00",
+ "green": "#00FF00",
+ "blue": "#0000FF",
+ "purple": "#800080",
+ "pink": "#FFC0CB",
+ "cyan": "#00FFFF",
+ "magenta": "#FF00FF",
+ "white": "#FFFFFF",
+ "black": "#000000",
+ "gray": "#808080",
+ "unknown": "#C0C0C0"
+ };
+
+ // Draw each row
+ this.colorData.forEach((rowData, rowIndex) => {
+ const y = rowIndex * rowHeight;
+
+ // Draw row background
+ ctx.fillStyle = rowIndex % 2 === 0 ? "#ffffff" : "#f8f8f8";
+ ctx.fillRect(0, y, canvasWidth, rowHeight);
+
+ // Draw row label
+ ctx.fillStyle = "#000000";
+ ctx.font = "12px Arial";
+ ctx.textAlign = "left";
+ ctx.fillText(`${rowData.label} (${rowData.note})`, 10, y + 20);
+
+ // Draw color segments
+ if (rowData.colorSegments && rowData.colorSegments.length > 0) {
+ let currentX = 150; // Start after label
+ const segmentHeight = 30;
+ const segmentY = y + 10;
+ const availableWidth = canvasWidth - 150 - 20; // Space for segments
+
+ // Calculate total duration for proportional sizing
+ const totalDuration = rowData.colorSegments.reduce((sum, segment) => sum + segment.duration, 0);
+
+ rowData.colorSegments.forEach((segment, segmentIndex) => {
+ // Calculate segment width proportional to its duration
+ const segmentWidth = Math.max(20, (segment.duration / totalDuration) * availableWidth);
+
+ // Draw color segment
+ ctx.fillStyle = colorMap[segment.color] || colorMap["unknown"];
+ ctx.fillRect(currentX, segmentY, segmentWidth, segmentHeight);
+
+ // Draw segment border
+ ctx.strokeStyle = "#333333";
+ ctx.lineWidth = 1;
+ ctx.strokeRect(currentX, segmentY, segmentWidth, segmentHeight);
+
+ // Draw color name if segment is wide enough
+ if (segmentWidth > 40) {
+ ctx.fillStyle = segment.color === "white" || segment.color === "yellow" ? "#000000" : "#ffffff";
+ ctx.font = "10px Arial";
+ ctx.textAlign = "center";
+ ctx.fillText(segment.color, currentX + segmentWidth/2, segmentY + segmentHeight/2 + 3);
+ }
+
+ // Draw duration text below
+ ctx.fillStyle = "#666666";
+ ctx.font = "8px Arial";
+ ctx.textAlign = "center";
+ ctx.fillText(`${Math.round(segment.duration)}ms`, currentX + segmentWidth/2, segmentY + segmentHeight + 12);
+
+ currentX += segmentWidth + 2; // Small gap between segments
+ });
+
+ } else {
+
+ // No colors detected
+ ctx.fillStyle = "#cccccc";
+ ctx.font = "12px Arial";
+ ctx.textAlign = "left";
+ ctx.fillText("No colors were detected", 150, y + 25);
+ }
+
+ // Draw the red lines
+ ctx.strokeStyle = "#dddddd";
+
+ ctx.lineWidth = 1;
+
+ ctx.beginPath();
+
+ ctx.moveTo(0, y + rowHeight);
+
+ ctx.lineTo(canvasWidth, y + rowHeight);
+ ctx.stroke();
+ });
+
+ // Draw column edge lines after all rows are drawn
+ this._drawColumnLines(ctx, canvasWidth, canvasHeight, 150, canvasWidth - 150 - 20);
+
+ // Add title
+ ctx.fillStyle = "#000000";
+ ctx.font = "bold 16px Arial";
+ ctx.textAlign = "center";
+ ctx.fillText("Color Detection Visualization", canvasWidth/2, -10);
+
+ // Convert canvas to PNG and download
+ canvas.toBlob((blob) => {
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement("a");
+ link.href = url;
+ link.download = `color_detection_${new Date().getTime()}.png`;
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ URL.revokeObjectURL(url);
+ }, "image/png");
+
+ // Play the music after visualization is generated
+ this.playColorMusicPolyphonic(this.colorData);
+ };
+
+ /**
+ * Plays all detected notes simultaneously, using filtered column boundaries.
+ * Only plays when color is NOT green.
+ * Updated to use same filtering logic as export (350ms minimum).
+ * @param {Array} colorData - The colorData array from scanning.
+ */
+ this.playColorMusicPolyphonic = async function(colorData) {
+ if (!this.synth) this._initAudio();
+
+ // Use the same boundary analysis and filtering as export
+ const columnBoundaries = this._analyzeColumnBoundaries();
+ const filteredBoundaries = this._filterSmallSegments(columnBoundaries);
+
+ // Build timeline using filtered boundaries instead of raw segments
+ let events = [];
+
+ // For each filtered time column, check which notes should play
+ for (let colIndex = 0; colIndex < filteredBoundaries.length - 1; colIndex++) {
+ const startTime = filteredBoundaries[colIndex];
+ const endTime = filteredBoundaries[colIndex + 1];
+ const duration = endTime - startTime;
+
+ // Check each row for non-green colors in this time range
+ colorData.forEach((rowData, rowIndex) => {
+ if (rowData.colorSegments && rowData.note) {
+ let currentTime = 0;
+ let hasNonGreenColor = false;
+
+ // Check if this time column overlaps with any non-green segments
+ for (const segment of rowData.colorSegments) {
+ const segmentStart = currentTime;
+ const segmentEnd = currentTime + segment.duration;
+
+ // Check if this segment overlaps with our time column
+ if (segmentStart < endTime && segmentEnd > startTime) {
+ // Calculate the actual overlap duration
+ const overlapStart = Math.max(segmentStart, startTime);
+ const overlapEnd = Math.min(segmentEnd, endTime);
+ const overlapDuration = overlapEnd - overlapStart;
+
+ // Only count as significant if overlap is substantial (>350ms)
+ // This prevents spillovers <350ms across blue lines from creating duplicate notes
+ if (overlapDuration > 350 && segment.color !== "green") {
+ hasNonGreenColor = true;
+ break;
+ } else if (overlapDuration <= 350 && segment.color !== "green") {
+ // Ignore small overlaps during playback
+ }
+ }
+ currentTime += segment.duration;
+ }
+
+ // If we found non-green color, add note on/off events
+ if (hasNonGreenColor) {
+ events.push({
+ time: startTime,
+ type: "on",
+ note: rowData.note,
+ rowIdx: rowIndex
+ });
+ events.push({
+ time: endTime,
+ type: "off",
+ note: rowData.note,
+ rowIdx: rowIndex
+ });
+ }
+ }
+ });
+ }
+
+ // Sort events by time
+ events.sort((a, b) => a.time - b.time);
+
+ // Track which notes are currently playing
+ let playingNotes = new Set();
+ let lastTime = 0;
+
+ for (let i = 0; i < events.length; i++) {
+ const evt = events[i];
+ const waitTime = evt.time - lastTime;
+ if (waitTime > 0) {
+ // Wait for the time until the next event
+ await new Promise(resolve => setTimeout(resolve, waitTime));
+ }
+ if (evt.type === "on") {
+ // Start note (if not already playing)
+ if (!playingNotes.has(evt.note)) {
+ this.synth.trigger(0, evt.note, 999, this.selectedInstrument, null, null, false, 0); // Long duration, will stop manually
+ playingNotes.add(evt.note);
+ }
+ } else if (evt.type === "off") {
+ // Stop note
+ this.synth.stopSound(0, this.selectedInstrument, evt.note);
+ playingNotes.delete(evt.note);
+ }
+ lastTime = evt.time;
+ }
+ // Ensure all notes are stopped at the end
+ playingNotes.forEach(note => {
+ this.synth.stopSound(0, this.selectedInstrument, note);
+ });
+ };
+
+}