debug-csv-encoding.ps1

# Debug CSV Encoding Issues
Write-Host "Debugging CSV Encoding Issues" -ForegroundColor Cyan
Write-Host "=============================" -ForegroundColor Cyan

$csvFile = "backend\Datasets\Diabetes.csv"

if (Test-Path $csvFile) {
    # Read raw content
    $csvContent = Get-Content $csvFile -Raw
    
    Write-Host "`nOriginal CSV Analysis:" -ForegroundColor Yellow
    Write-Host "Total length: $($csvContent.Length) characters" -ForegroundColor White
    
    # Count different line ending types
    $crlfCount = ([regex]::Matches($csvContent, "`r`n")).Count
    $lfCount = ([regex]::Matches($csvContent, "(?<!`r)`n")).Count
    $crCount = ([regex]::Matches($csvContent, "`r(?!`n)")).Count
    
    Write-Host "Line endings found:" -ForegroundColor White
    Write-Host "  - CRLF (\\r\\n): $crlfCount" -ForegroundColor Gray
    Write-Host "  - LF (\\n): $lfCount" -ForegroundColor Gray
    Write-Host "  - CR (\\r): $crCount" -ForegroundColor Gray
    
    # Check for other control characters
    $controlChars = [regex]::Matches($csvContent, "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]")
    Write-Host "  - Other control chars: $($controlChars.Count)" -ForegroundColor Gray
    
    if ($controlChars.Count -gt 0) {
        Write-Host "Control characters found at positions:" -ForegroundColor Red
        foreach ($match in $controlChars | Select-Object -First 10) {
            $char = $match.Value
            $ascii = [int][char]$char
            Write-Host "    Position $($match.Index): ASCII $ascii (0x$($ascii.ToString('X2')))" -ForegroundColor Red
        }
    }
    
    # Show first few lines in hex
    $firstLine = ($csvContent -split "`n")[0]
    Write-Host "`nFirst line in hex:" -ForegroundColor Yellow
    $bytes = [System.Text.Encoding]::UTF8.GetBytes($firstLine)
    $hex = ($bytes | ForEach-Object { $_.ToString("X2") }) -join " "
    Write-Host $hex -ForegroundColor Gray
    
    # Test cleaning
    Write-Host "`nCleaning test:" -ForegroundColor Yellow
    $cleanedCsv = $csvContent -replace "`r`n", "`n" -replace "`r", "`n"
    $cleanedCsv = $cleanedCsv -replace "[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", ""
    
    Write-Host "After cleaning: $($cleanedCsv.Length) characters" -ForegroundColor White
    
    # Test base64 encoding
    Write-Host "`nBase64 encoding test:" -ForegroundColor Yellow
    
    # Original (problematic)
    $originalBytes = [System.Text.Encoding]::UTF8.GetBytes($csvContent)
    $originalBase64 = [System.Convert]::ToBase64String($originalBytes)
    Write-Host "Original base64 length: $($originalBase64.Length)" -ForegroundColor Red
    
    # Cleaned
    $cleanedBytes = [System.Text.Encoding]::UTF8.GetBytes($cleanedCsv)
    $cleanedBase64 = [System.Convert]::ToBase64String($cleanedBytes)
    Write-Host "Cleaned base64 length: $($cleanedBase64.Length)" -ForegroundColor Green
    
    # Test JSON encoding
    Write-Host "`nJSON encoding test:" -ForegroundColor Yellow
    
    try {
        $testObj = @{ csv_data = $originalBase64 }
        $testJson = $testObj | ConvertTo-Json -Compress
        Write-Host "Original base64 in JSON: SUCCESS (length: $($testJson.Length))" -ForegroundColor Green
    } catch {
        Write-Host "Original base64 in JSON: FAILED - $($_.Exception.Message)" -ForegroundColor Red
    }
    
    try {
        $testObj = @{ csv_data = $cleanedBase64 }
        $testJson = $testObj | ConvertTo-Json -Compress
        Write-Host "Cleaned base64 in JSON: SUCCESS (length: $($testJson.Length))" -ForegroundColor Green
    } catch {
        Write-Host "Cleaned base64 in JSON: FAILED - $($_.Exception.Message)" -ForegroundColor Red
    }
    
} else {
    Write-Host "CSV file not found: $csvFile" -ForegroundColor Red
}

Write-Host "`nDebugging complete!" -ForegroundColor Cyan