diff --git a/.github/scripts/commit_prefix_check.py b/.github/scripts/commit_prefix_check.py index a345a60eced..60abbcfdab5 100644 --- a/.github/scripts/commit_prefix_check.py +++ b/.github/scripts/commit_prefix_check.py @@ -16,6 +16,7 @@ import re import sys from git import Repo +from git.exc import GitCommandError repo = Repo(".") @@ -170,7 +171,15 @@ def validate_commit(commit): return False, "Missing Signed-off-by line" # Determine expected prefixes + build option flag - files = commit.stats.files.keys() + try: + files = commit.stats.files.keys() + except GitCommandError as e: + return False, ( + f"Could not inspect files changed by commit {commit.hexsha[:10]}: {e}\n" + "The repository checkout is likely missing commit parent history. " + "Use a full-depth checkout for commit-prefix validation." + ) + expected, build_optional = infer_prefix_from_paths(files) # When no prefix can be inferred (docs/tools), allow anything diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml index 5174c0f5cfd..11c70617ba0 100644 --- a/.github/workflows/call-build-images.yaml +++ b/.github/workflows/call-build-images.yaml @@ -413,6 +413,26 @@ jobs: username: ${{ inputs.username }} password: ${{ secrets.token }} + # https://github.com/actions/runner-images/issues/12199 + # https://github.com/moby/moby/issues/48093 + # https://support.microsoft.com/en-us/topic/gettemppath-changes-in-windows-february-cumulative-update-preview-4cc631fb-9d97-4118-ab6d-f643cd0a7259#ID0EDF + - name: Change system temp directory to the drive having sufficient disk space + id: relocate-system-temp-dir + run: | + $ErrorActionPreference = "Stop" + if (Test-Path -Path "D:\") { + $newSystemTempDir = "D:\SystemTemp" + New-Item -ItemType Directory -Path "${newSystemTempDir}" -Force + $acl = New-Object System.Security.AccessControl.DirectorySecurity + $acl.SetSecurityDescriptorSddlForm("O:SYG:SYD:PAI(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)") + Set-Acl "${newSystemTempDir}" -AclObject ${acl} + [Environment]::SetEnvironmentVariable("SYSTEMTEMP", "${newSystemTempDir}", [EnvironmentVariableTarget]::Machine) + Restart-Service docker + } else { + Write-Warning "Drive D doesn't exist, so there is no disk to relocate system temp directory" + } + shell: pwsh + - name: Pull the last release image to speed up the build with a cache continue-on-error: true run: | diff --git a/.github/workflows/call-build-windows.yaml b/.github/workflows/call-build-windows.yaml index 94897756786..27309be2244 100644 --- a/.github/workflows/call-build-windows.yaml +++ b/.github/workflows/call-build-windows.yaml @@ -72,7 +72,7 @@ jobs: shell: bash call-build-windows-package: - runs-on: windows-latest + runs-on: ${{ matrix.config.os }} environment: ${{ inputs.environment }} needs: - call-build-windows-get-meta @@ -85,16 +85,19 @@ jobs: cmake_additional_opt: "" vcpkg_triplet: x86-windows-static cmake_version: "3.31.6" + os: windows-latest - name: "Windows 64bit" arch: x64 cmake_additional_opt: "" vcpkg_triplet: x64-windows-static cmake_version: "3.31.6" + os: windows-latest - name: "Windows 64bit (Arm64)" arch: amd64_arm64 cmake_additional_opt: "-DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SYSTEM_VERSION=10.0 -DCMAKE_SYSTEM_PROCESSOR=ARM64" vcpkg_triplet: arm64-windows-static cmake_version: "3.31.6" + os: windows-11-arm permissions: contents: read # Default environment variables can be overridden below. To prevent library pollution - without this other random libraries may be found on the path leading to failures. @@ -106,6 +109,25 @@ jobs: with: ref: ${{ inputs.ref }} + - name: Disk space before initial cleanup + run: Get-PSDrive C + shell: pwsh + + - name: Free up disk space + run: | + Remove-Item -Recurse -Force "C:\Windows\Temp\*" ` + -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "$env:TEMP\*" ` + -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force ` + "C:\Users\runneradmin\AppData\Local\Temp\*" ` + -ErrorAction SilentlyContinue + shell: pwsh + + - name: Disk space after initial cleanup + run: Get-PSDrive C + shell: pwsh + - name: Get dependencies run: | Invoke-WebRequest -OutFile winflexbison.zip $env:WINFLEXBISON @@ -123,11 +145,71 @@ jobs: with: arch: ${{ matrix.config.arch }} + - name: Disk space before Chocolatey cleanup + run: Get-PSDrive C + shell: pwsh + + - name: Free up disk space before Chocolatey install + run: | + Remove-Item -Recurse -Force "C:\Windows\Temp\*" ` + -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force "$env:TEMP\*" ` + -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force ` + "C:\Users\runneradmin\AppData\Local\Temp\*" ` + -ErrorAction SilentlyContinue + shell: pwsh + + - name: Disk space after Chocolatey cleanup + run: Get-PSDrive C + shell: pwsh + - name: Get gzip command and nsis w/ chocolatey uses: crazy-max/ghaction-chocolatey@v3 with: args: install gzip nsis -y + - name: Get WiX Toolset w/ chocolatey + if: ${{ matrix.config.arch == 'amd64_arm64' }} + uses: crazy-max/ghaction-chocolatey@dff3862348493b11fba2fbc49147b6d2dfe09b66 # v4.0.0 + with: + args: install wixtoolset -y + + - name: Export WiX Toolset path + if: ${{ matrix.config.arch == 'amd64_arm64' }} + run: | + $wix = [Environment]::GetEnvironmentVariable("WIX", "Machine") + if (-not $wix) { + $wix = [Environment]::GetEnvironmentVariable("WIX", "Process") + } + if (-not $wix) { + $candidates = @( + "${env:ProgramFiles(x86)}", + "${env:ProgramFiles}" + ) | Where-Object { $_ -and (Test-Path $_) } + foreach ($candidate in $candidates) { + $wix = Get-ChildItem -Path $candidate -Directory -Filter "WiX Toolset v*" | + Sort-Object Name -Descending | + Select-Object -First 1 -ExpandProperty FullName + if ($wix) { + break + } + } + } + if (-not $wix) { + throw "WiX Toolset installation path could not be located" + } + + $wixBin = Join-Path $wix "bin" + if (!(Test-Path (Join-Path $wixBin "candle.exe")) -or + !(Test-Path (Join-Path $wixBin "light.exe"))) { + throw "WiX Toolset candle.exe/light.exe could not be located in $wixBin" + } + + "WIX=$wix" | Out-File -FilePath $env:GITHUB_ENV -Append + $wixBin | Out-File -FilePath $env:GITHUB_PATH -Append + shell: pwsh + # http://man7.org/linux/man-pages/man1/date.1.html - name: Get Date id: get-date diff --git a/.github/workflows/call-windows-unit-tests.yaml b/.github/workflows/call-windows-unit-tests.yaml index 5b5bbf56c07..e820c73deb9 100644 --- a/.github/workflows/call-windows-unit-tests.yaml +++ b/.github/workflows/call-windows-unit-tests.yaml @@ -140,7 +140,6 @@ jobs: -D FLB_WITHOUT_flb-rt-out_forward=On ` -D FLB_WITHOUT_flb-rt-in_disk=On ` -D FLB_WITHOUT_flb-rt-in_proc=On ` - -D FLB_WITHOUT_flb-it-parser=On ` -D FLB_WITHOUT_flb-it-unit_sizes=On ` -D FLB_WITHOUT_flb-it-network=On ` -D FLB_WITHOUT_flb-it-pack=On ` diff --git a/.github/workflows/commit-lint.yaml b/.github/workflows/commit-lint.yaml index e3100ab99da..346791401f6 100644 --- a/.github/workflows/commit-lint.yaml +++ b/.github/workflows/commit-lint.yaml @@ -15,7 +15,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 with: - fetch-depth: 50 # needed to see ancestor commits + fetch-depth: 0 # commit-prefix validation needs complete parent history - name: Fetch base branch if: github.event_name == 'pull_request' diff --git a/.github/workflows/cron-trivy.yaml b/.github/workflows/cron-trivy.yaml deleted file mode 100644 index 3734a54288c..00000000000 --- a/.github/workflows/cron-trivy.yaml +++ /dev/null @@ -1,87 +0,0 @@ ---- -# Separate action to allow us to initiate manually and run regularly -name: Trivy security analysis of latest containers - -# Run on every push to master, or weekly. -# Allow users to trigger an asynchronous run anytime too. -on: - push: - branches: [master] - schedule: - # 13:44 on Thursday - - cron: 44 13 * * 4 - workflow_dispatch: - -jobs: - # Run Trivy on the latest container and update the security code scanning results tab. - trivy-latest: - # Matrix job that pulls the latest image for each supported architecture via the multi-arch latest manifest. - # We then re-tag it locally to ensure that when Trivy runs it does not pull the latest for the wrong architecture. - name: ${{ matrix.arch }} container scan - runs-on: [ ubuntu-latest ] - continue-on-error: true - strategy: - fail-fast: false - # Matrix of architectures to test along with their local tags for special character substitution - matrix: - # The architecture for the container runtime to pull. - arch: [ linux/amd64, linux/arm64, linux/arm/v7 ] - # In a few cases we need the arch without slashes so provide a descriptive extra field for that. - # We could also extract or modify this via a regex but this seemed simpler and easier to follow. - include: - - arch: linux/amd64 - local_tag: x86_64 - - arch: linux/arm64 - local_tag: arm64 - - arch: linux/arm/v7 - local_tag: arm32 - steps: - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Pull the image for the architecture we're testing - run: | - docker pull --platform ${{ matrix.arch }} fluent/fluent-bit:latest - - - name: Tag locally to ensure we do not pull wrong architecture - run: | - docker tag fluent/fluent-bit:latest local/fluent-bit:${{ matrix.local_tag }} - - # Deliberately chosen master here to keep up-to-date. - - name: Run Trivy vulnerability scanner for any major issues - uses: aquasecurity/trivy-action@master - with: - image-ref: local/fluent-bit:${{ matrix.local_tag }} - # Filter out any that have no current fix. - ignore-unfixed: true - # Only include major issues. - severity: CRITICAL,HIGH - format: template - template: '@/contrib/sarif.tpl' - output: trivy-results-${{ matrix.local_tag }}.sarif - - # Show all detected issues. - # Note this will show a lot more, including major un-fixed ones. - - name: Run Trivy vulnerability scanner for local output - uses: aquasecurity/trivy-action@master - with: - image-ref: local/fluent-bit:${{ matrix.local_tag }} - format: table - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: trivy-results-${{ matrix.local_tag }}.sarif - category: ${{ matrix.arch }} container - wait-for-processing: true - - # In case we need to analyse the uploaded files for some reason. - - name: Detain results for debug if needed - uses: actions/upload-artifact@v6 - with: - name: trivy-results-${{ matrix.local_tag }}.sarif - path: trivy-results-${{ matrix.local_tag }}.sarif - if-no-files-found: error diff --git a/.github/workflows/pr-image-tests.yaml b/.github/workflows/pr-image-tests.yaml index 6ffd94a507d..06f7116567d 100644 --- a/.github/workflows/pr-image-tests.yaml +++ b/.github/workflows/pr-image-tests.yaml @@ -108,6 +108,26 @@ jobs: flavor: | suffix=-windows-${{ matrix.windows-base-version }} + # https://github.com/actions/runner-images/issues/12199 + # https://github.com/moby/moby/issues/48093 + # https://support.microsoft.com/en-us/topic/gettemppath-changes-in-windows-february-cumulative-update-preview-4cc631fb-9d97-4118-ab6d-f643cd0a7259#ID0EDF + - name: Change system temp directory to the drive having sufficient disk space + id: relocate-system-temp-dir + run: | + $ErrorActionPreference = "Stop" + if (Test-Path -Path "D:\") { + $newSystemTempDir = "D:\SystemTemp" + New-Item -ItemType Directory -Path "${newSystemTempDir}" -Force + $acl = New-Object System.Security.AccessControl.DirectorySecurity + $acl.SetSecurityDescriptorSddlForm("O:SYG:SYD:PAI(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)") + Set-Acl "${newSystemTempDir}" -AclObject ${acl} + [Environment]::SetEnvironmentVariable("SYSTEMTEMP", "${newSystemTempDir}", [EnvironmentVariableTarget]::Machine) + Restart-Service docker + } else { + Write-Warning "Drive D doesn't exist, so there is no disk to relocate system temp directory" + } + shell: pwsh + - name: Build the windows images id: build run: | diff --git a/.github/workflows/pr-windows-build.yaml b/.github/workflows/pr-windows-build.yaml index 110d85fda64..9be1e6daa7c 100644 --- a/.github/workflows/pr-windows-build.yaml +++ b/.github/workflows/pr-windows-build.yaml @@ -22,8 +22,8 @@ on: - '**.h' - '**.c' - '**.windows' - - './conf/**' - - './cmake/**' + - 'conf/**' + - 'cmake/**' types: - opened - reopened diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index a51e5b7d4e6..5e209dec99e 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -22,6 +22,7 @@ on: - 'examples/**' branches: - master + - 4.2 - 4.1 - 4.0 - 3.2 @@ -55,6 +56,7 @@ jobs: - "-DFLB_SIMD=Off" - "-DFLB_ARROW=On" - "-DFLB_COMPILER_STRICT_POINTER_TYPES=On" + - "-DFLB_AVRO_ENCODER=On -DCMAKE_POLICY_VERSION_MINIMUM=3.5" cmake_version: - "3.31.6" compiler: @@ -77,6 +79,10 @@ jobs: compiler: cc: clang cxx: clang++ + - flb_option: "-DFLB_AVRO_ENCODER=On -DCMAKE_POLICY_VERSION_MINIMUM=3.5" + compiler: + cc: clang + cxx: clang++ permissions: contents: read steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index b790cf42a69..495960b088b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) # Fluent Bit Version set(FLB_VERSION_MAJOR 4) set(FLB_VERSION_MINOR 2) -set(FLB_VERSION_PATCH 3) +set(FLB_VERSION_PATCH 7) set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -658,8 +658,8 @@ if(FLB_AVRO_ENCODER) # jansson option(JANSSON_BUILD_DOCS OFF) option(JANSSON_EXAMPLES OFF) - option(JANSSON_WITHOUT_TESTS ON) option(JANSSON_BUILD_SHARED_LIBS OFF) + set(JANSSON_WITHOUT_TESTS ON CACHE BOOL "" FORCE) add_subdirectory(${FLB_PATH_LIB_JANSSON}) #avro diff --git a/CODEOWNERS b/CODEOWNERS index 317fb7c19af..1b09990afbf 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -63,7 +63,7 @@ /plugins/out_datadog @nokute78 @edsiper /plugins/out_es @pettitwesley @edsiper /plugins/out_pgsql @sxd -/plugins/out_stackdriver @braydonk @jefferbrecht @jeffluoo +/plugins/out_stackdriver @jeffluoo # AWS Plugins /plugins/out_s3 @fluent/aws-fluent-bit-maintainers @@ -85,8 +85,8 @@ # Google test code # -------------- -/tests/runtime/out_stackdriver.c @braydonk @jefferbrecht @jeffluoo -/tests/runtime/data/stackdriver @braydonk @jefferbrecht @jeffluoo +/tests/runtime/out_stackdriver.c @jeffluoo +/tests/runtime/data/stackdriver @jeffluoo # Devcontainer /.devcontainer @patrick-stephens @niedbalski @edsiper diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 6348c5f005f..e0847d828d8 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -13,7 +13,7 @@ # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/ # Set this to the current release version: it gets done so as part of the release. -ARG RELEASE_VERSION=4.2.3 +ARG RELEASE_VERSION=4.2.7 # For multi-arch builds - assumption is running on an AMD64 host FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32 diff --git a/dockerfiles/Dockerfile.windows b/dockerfiles/Dockerfile.windows index e40800f2535..52c5e11143d 100644 --- a/dockerfiles/Dockerfile.windows +++ b/dockerfiles/Dockerfile.windows @@ -41,13 +41,21 @@ RUN $msvs_build_tools_dist_name=\"vs_buildtools.exe\"; ` Invoke-WebRequest -OutFile \"${msvs_build_tools_dist}\" \"${msvs_build_tools_dist_url}\"; ` Invoke-WebRequest -OutFile \"${msvs_build_tools_channel}\" \"${msvs_build_tools_channel_url}\"; ` Write-Host \"Installing Visual Studio Build Tools into ${env:MSVS_HOME}...\"; ` - Start-Process \"${msvs_build_tools_dist}\" ` + $p = Start-Process \"${msvs_build_tools_dist}\" ` -ArgumentList '--quiet ', '--wait ', '--norestart ', '--nocache', ` \"--installPath ${env:MSVS_HOME}\", ` \"--channelUri ${msvs_build_tools_channel}\", ` \"--installChannelUri ${msvs_build_tools_channel}\", ` '--add Microsoft.VisualStudio.Workload.VCTools', ` - '--includeRecommended' -NoNewWindow -Wait; ` + '--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64', ` + '--add Microsoft.VisualStudio.Component.Windows10SDK.19041' ` + -NoNewWindow -Wait -PassThru; ` + if (${p}.ExitCode -ne 0 -and ${p}.ExitCode -ne 3010) { ` + throw \"Visual Studio Build Tools installer failed with exit code $(${p}.ExitCode)\" ` + }; ` + if (-not (Test-Path \"${env:MSVS_HOME}\VC\Auxiliary\Build\vcvars64.bat\")) { ` + throw \"Visual Studio Build Tools installation is incomplete: ${env:MSVS_HOME}\VC\Auxiliary\Build\vcvars64.bat not found\" ` + }; ` Remove-Item -Force \"${msvs_build_tools_dist}\"; ` Remove-Item -Path \"${msvs_build_tools_channel}\" -Force; @@ -83,6 +91,26 @@ RUN if ([System.Version] \"${env:CMAKE_VERSION}\" -ge [System.Version] \"3.20.0\ Write-Host \"${env:PATH}\"; ` [Environment]::SetEnvironmentVariable(\"PATH\", \"${env:PATH}\", [EnvironmentVariableTarget]::Machine); +ENV NINJA_HOME="C:\ninja" +ARG NINJA_VERSION="1.13.2" +ARG NINJA_URL="https://github.com/ninja-build/ninja/releases/download" + +RUN $ninja_dist_name=\"ninja-win.zip\"; ` + $ninja_dist=\"${env:TMP}\${ninja_dist_name}\"; ` + $ninja_download_url=\"${env:NINJA_URL}/v${env:NINJA_VERSION}/${ninja_dist_name}\"; ` + Write-Host \"Downloading Ninja...\"; ` + Write-Host \"${ninja_download_url} -> ${ninja_dist}\"; ` + Invoke-WebRequest -OutFile \"${ninja_dist}\" \"${ninja_download_url}\"; ` + New-Item -Path \"${env:NINJA_HOME}\" -ItemType \"directory\"; ` + Write-Host \"Extracting Ninja...\"; ` + Write-Host \"${ninja_dist} -> ${env:NINJA_HOME}\"; ` + Expand-Archive \"${ninja_dist}\" -Destination \"${env:NINJA_HOME}\"; ` + Remove-Item -Force \"${ninja_dist}\"; ` + $env:PATH=\"${env:PATH};${env:NINJA_HOME}\"; ` + Write-Host \"Setting PATH...\"; ` + Write-Host \"${env:PATH}\"; ` + [Environment]::SetEnvironmentVariable(\"PATH\", \"${env:PATH}\", [EnvironmentVariableTarget]::Machine); + ENV WIN_FLEX_BISON_VERSION="2.5.22" ` WIN_FLEX_BISON_HOME="C:\WinFlexBison" ` WIN_FLEX_BISON_DOWNLOAD_URL="https://github.com/lexxmark/winflexbison/releases/download" @@ -142,7 +170,8 @@ RUN $vcpkg_dist_base_name=\"vcpkg-${env:VCPKG_VERSION}\"; ` # Ensure we only attempt to build release and static linking ENV VCPKG_BUILD_TYPE=release ` - VCPKG_LIBRARY_LINKAGE=static + VCPKG_LIBRARY_LINKAGE=static ` + VCPKG_VISUAL_STUDIO_PATH="C:\BuildTools" # Install dependencies and clean temporary files to save space RUN vcpkg install --recurse openssl --triplet x64-windows-static; ` @@ -169,7 +198,7 @@ COPY . /src/ ARG BUILD_PARALLEL=1 SHELL ["cmd", "/S", "/C"] RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && ` - cmake -G "NMake Makefiles" ` + cmake -G "Ninja" ` -DOPENSSL_ROOT_DIR='C:\dev\vcpkg\packages\openssl_x64-windows-static' ` -DFLB_LIBYAML_DIR='C:\dev\vcpkg\packages\libyaml_x64-windows-static' ` -DFLB_SIMD=On ` @@ -179,7 +208,7 @@ RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && ` -DFLB_DEBUG=Off ` -DFLB_RELEASE=On ` ..\ && ` - cmake --build . --config Release -j "%BUILD_PARALLEL%" + cmake --build . -j "%BUILD_PARALLEL%" SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] diff --git a/fluent-bit-4.2.3.bb b/fluent-bit-4.2.7.bb similarity index 99% rename from fluent-bit-4.2.3.bb rename to fluent-bit-4.2.7.bb index 1ceef87f0b1..cca31804f14 100644 --- a/fluent-bit-4.2.3.bb +++ b/fluent-bit-4.2.7.bb @@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93" SECTION = "net" PR = "r0" -PV = "4.2.3" +PV = "4.2.7" SRCREV = "v${PV}" SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1" diff --git a/include/fluent-bit/flb_avro.h b/include/fluent-bit/flb_avro.h index 581baf622d5..abb10a35b99 100644 --- a/include/fluent-bit/flb_avro.h +++ b/include/fluent-bit/flb_avro.h @@ -31,7 +31,7 @@ #define MEMORY_POOL_MINIMUM_SIZE sizeof(void *) struct flb_avro_fields { - flb_sds_t schema_id; + int32_t schema_id; flb_sds_t schema_str; }; diff --git a/include/fluent-bit/flb_crypto.h b/include/fluent-bit/flb_crypto.h index e406388e457..1a0ae9def30 100644 --- a/include/fluent-bit/flb_crypto.h +++ b/include/fluent-bit/flb_crypto.h @@ -24,7 +24,9 @@ #include #include #include +#ifndef OPENSSL_NO_ENGINE #include +#endif #include diff --git a/include/fluent-bit/flb_hash.h b/include/fluent-bit/flb_hash.h index 8c1c7bdf25e..78e4f9e363d 100644 --- a/include/fluent-bit/flb_hash.h +++ b/include/fluent-bit/flb_hash.h @@ -51,7 +51,9 @@ #include #include #include +#ifndef OPENSSL_NO_ENGINE #include +#endif #include diff --git a/include/fluent-bit/flb_hmac.h b/include/fluent-bit/flb_hmac.h index 73ad2439070..49fc4979017 100644 --- a/include/fluent-bit/flb_hmac.h +++ b/include/fluent-bit/flb_hmac.h @@ -23,7 +23,9 @@ #include #include #include +#ifndef OPENSSL_NO_ENGINE #include +#endif #include struct flb_hmac { diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 2b3241d4d3c..454f818fd4c 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -1188,8 +1188,10 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, static inline void flb_output_return(int ret, struct flb_coro *co) { int n; int pipe_fd; + int effective_records; uint32_t set; uint64_t val; + size_t effective_bytes; struct flb_task *task; struct flb_output_flush *out_flush; struct flb_output_instance *o_ins; @@ -1199,8 +1201,21 @@ static inline void flb_output_return(int ret, struct flb_coro *co) { o_ins = out_flush->o_ins; task = out_flush->task; + effective_records = 0; + effective_bytes = 0; + if (task->event_chunk != NULL) { + effective_records = task->event_chunk->total_events; + effective_bytes = task->event_chunk->size; + } + + if (out_flush->processed_event_chunk != NULL) { + effective_records = out_flush->processed_event_chunk->total_events; + effective_bytes = out_flush->processed_event_chunk->size; + } + flb_task_acquire_lock(task); + flb_task_set_route_metrics(task, o_ins, effective_records, effective_bytes); flb_task_deactivate_route(task, o_ins); flb_task_release_lock(task); diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h index 62770a97109..10272b1cab0 100644 --- a/include/fluent-bit/flb_parser.h +++ b/include/fluent-bit/flb_parser.h @@ -49,6 +49,8 @@ struct flb_parser { char *time_key; /* field name that contains the time */ int time_offset; /* fixed UTC offset */ int time_system_timezone; /* use the system timezone as a fallback */ + char *time_zone; /* IANA timezone for naive timestamps */ + void *time_zone_data; /* parsed native timezone data */ int time_keep; /* keep time field */ int time_strict; /* parse time field strictly */ int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */ @@ -91,6 +93,8 @@ static inline time_t flb_parser_tm2time(const struct flb_tm *src, return res; } +time_t flb_parser_tm2time_parser(const struct flb_tm *src, + struct flb_parser *parser); struct flb_parser *flb_parser_create(const char *name, const char *format, const char *p_regex, @@ -105,6 +109,22 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, int types_len, struct mk_list *decoders, struct flb_config *config); +struct flb_parser *flb_parser_create_with_time_zone(const char *name, + const char *format, + const char *p_regex, + int skip_empty, + const char *time_fmt, + const char *time_key, + const char *time_offset, + int time_keep, + int time_strict, + int time_system_timezone, + const char *time_zone, + int logfmt_no_bare_keys, + struct flb_parser_types *types, + int types_len, + struct mk_list *decoders, + struct flb_config *config); int flb_parser_conf_file_stat(const char *file, struct flb_config *config); int flb_parser_conf_file(const char *file, struct flb_config *config); int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, diff --git a/include/fluent-bit/flb_plugin_proxy.h b/include/fluent-bit/flb_plugin_proxy.h index f5c6bbce262..ca224437a11 100644 --- a/include/fluent-bit/flb_plugin_proxy.h +++ b/include/fluent-bit/flb_plugin_proxy.h @@ -40,6 +40,7 @@ struct flb_plugin_proxy_def { int flags; char *name; /* plugin short name */ char *description; /* plugin description */ + int event_type; /* event type (logs/metrics/traces) */ }; /* Proxy context */ diff --git a/include/fluent-bit/flb_task.h b/include/fluent-bit/flb_task.h index 6d93ba1535e..683f9f9207d 100644 --- a/include/fluent-bit/flb_task.h +++ b/include/fluent-bit/flb_task.h @@ -57,6 +57,8 @@ struct flb_task_route { int status; + int records; + size_t bytes; struct flb_output_instance *out; struct mk_list _head; }; @@ -257,6 +259,54 @@ static FLB_INLINE void flb_task_set_route_status( } } +static FLB_INLINE void flb_task_set_route_metrics( + struct flb_task *task, + struct flb_output_instance *o_ins, + int records, + size_t bytes) +{ + struct mk_list *iterator; + struct flb_task_route *route; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + route->records = records; + route->bytes = bytes; + break; + } + } +} + +static FLB_INLINE int flb_task_get_route_metrics( + struct flb_task *task, + struct flb_output_instance *o_ins, + int *records, + size_t *bytes) +{ + struct mk_list *iterator; + struct flb_task_route *route; + + mk_list_foreach(iterator, &task->routes) { + route = mk_list_entry(iterator, struct flb_task_route, _head); + + if (route->out == o_ins) { + if (records != NULL) { + *records = route->records; + } + + if (bytes != NULL) { + *bytes = route->bytes; + } + + return 0; + } + } + + return -1; +} + static FLB_INLINE void flb_task_activate_route( struct flb_task *task, diff --git a/include/fluent-bit/flb_time.h b/include/fluent-bit/flb_time.h index 5205446a815..1bb3b954438 100644 --- a/include/fluent-bit/flb_time.h +++ b/include/fluent-bit/flb_time.h @@ -115,6 +115,10 @@ int flb_time_msgpack_to_time(struct flb_time *time, msgpack_object *obj); int flb_time_pop_from_mpack(struct flb_time *time, mpack_reader_t *reader); int flb_time_pop_from_msgpack(struct flb_time *time, msgpack_unpacked *upk, msgpack_object **map); +const char *flb_time_windows_zone_to_iana(const char *windows_zone); +const char *flb_time_iana_zone_to_windows(const char *iana_zone); +int flb_time_windows_zone_to_utc_offset(const char *windows_zone, long *offset); +int flb_time_iana_zone_to_utc_offset(const char *iana_zone, long *offset); long flb_time_tz_offset_to_second(); #endif /* FLB_TIME_H */ diff --git a/include/fluent-bit/flb_version.h.in b/include/fluent-bit/flb_version.h.in index ef1984a4b93..cf55f2186fd 100644 --- a/include/fluent-bit/flb_version.h.in +++ b/include/fluent-bit/flb_version.h.in @@ -2,7 +2,7 @@ /* Fluent Bit * ========== - * Copyright (C) 2015-2025 The Fluent Bit Authors + * Copyright (C) 2015-2026 The Fluent Bit Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,7 +81,7 @@ static inline void flb_version_banner() fprintf(stderr, "%sFluent Bit v%s%s\n", bold_color, FLB_VERSION_STR, reset_color); #endif - fprintf(stderr, "* %sCopyright (C) 2015-2025 The Fluent Bit Authors%s\n", + fprintf(stderr, "* %sCopyright (C) 2015-2026 The Fluent Bit Authors%s\n", copyright_color, reset_color); fprintf(stderr, "* Fluent Bit is a CNCF graduated project under the " "Fluent organization\n"); diff --git a/include/fluent-bit/multiline/flb_ml.h b/include/fluent-bit/multiline/flb_ml.h index 9824c20e2e2..c9c10e0ee43 100644 --- a/include/fluent-bit/multiline/flb_ml.h +++ b/include/fluent-bit/multiline/flb_ml.h @@ -112,6 +112,10 @@ struct flb_ml_stream_group { msgpack_sbuffer mp_md_sbuf; /* temporary msgpack buffer */ msgpack_packer mp_md_pck; /* temporary msgpack packer */ + /* Metadata snapshots (deep-copied) to avoid msgpack pack/unpack churn */ + struct mk_list metadata_objects; /* list of deep-copied msgpack_object maps */ + int metadata_objects_initialized; + msgpack_sbuffer mp_sbuf; /* temporary msgpack buffer */ msgpack_packer mp_pck; /* temporary msgpack packer */ struct flb_time mp_time; /* multiline time parsed from first line */ @@ -370,6 +374,10 @@ int flb_ml_flush_stdout(struct flb_ml_parser *parser, struct flb_ml_stream *mst, void *data, char *buf_data, size_t buf_size); +int flb_ml_stream_group_add_metadata(struct flb_ml_stream_group *group, + msgpack_object *metadata); +void flb_ml_stream_group_purge_metadata(struct flb_ml_stream_group *group); + #include "flb_ml_mode.h" #endif diff --git a/lib/monkey/AGENTS.md b/lib/monkey/AGENTS.md new file mode 100644 index 00000000000..110352cf173 --- /dev/null +++ b/lib/monkey/AGENTS.md @@ -0,0 +1,184 @@ +# AGENTS + +This file is the local operating guide for agents working in this repository. +It focuses on two things: + +- how the Monkey source tree is organized +- how commits are written in the existing Git history + +## Repository map + +Monkey is a small HTTP server written in C. The tree is split by subsystem. + +- `mk_core/` + Core utilities used by the server and plugins. + Includes memory helpers, strings, files, thread helpers, event loops, + I/O vectors, config parsing, and generic utilities. + +- `mk_server/` + The HTTP server implementation. + This is where connection handling, request parsing, header generation, + virtual hosts, MIME resolution, scheduler integration, streams, plugins, + and server lifecycle live. + +- `mk_bin/` + The standalone `monkey` executable entrypoints and signal handling. + +- `include/monkey/` + Public and internal headers for the core, server, parser, plugins, + config, events, streams, and API types. + +- `plugins/` + Optional server features. + Examples in this tree include `liana`, `mandril`, `dirlisting`, `cgi`, + `fastcgi`, `auth`, `logger`, `tls`, and `cheetah`. + +- `api/` + Small API-focused programs and tests. + +- `test/` + Native unit/integration-style test targets used by CMake. + +- `fuzz/` + Fuzzing entrypoints and helpers for parser and request handling. + +- `conf/` + Config templates installed or copied at build/install time. + +- `htdocs/` + Default static site content used by the standalone server. + +- `cmake/` + CMake helper modules and build logic. + +- `deps/` + Bundled third-party dependencies such as regex, rbtree, libco, + and mbedtls sources. + +- `qa/` + Extra request fixtures and local QA artifacts. + +## Main runtime flow + +When debugging behavior, the usual path is: + +1. `mk_bin/monkey.c` + Starts the binary. +2. `mk_server/monkey.c`, `mk_server/mk_server.c` + Initializes the server and worker threads. +3. `mk_server/mk_scheduler.c` + Drives socket events into protocol handlers. +4. `mk_server/mk_http.c` + Owns HTTP session lifecycle, request preparation, response handling, + range parsing, file serving, keepalive, and teardown. +5. `mk_server/mk_http_parser.c` + Parses the request line, headers, body state, and chunked transfer coding. +6. `mk_server/mk_header.c`, `mk_server/mk_stream.c` + Build and send responses. + +Useful supporting code: + +- `mk_server/mk_vhost.c` + Virtual host lookup, per-vhost file descriptor table. +- `mk_server/mk_mimetype.c` + File extension to MIME mapping. +- `mk_server/mk_user.c` + `~user` URI handling. +- `mk_server/mk_plugin.c` + Plugin registration and API exposure. +- `mk_core/mk_event_*.c` + Backend-specific event loop implementations. + +## Build and verification + +Typical local build entrypoint: + +```bash +cmake --build build +``` + +If a fresh build tree is needed, inspect `CMakeLists.txt` and the generated +`build/` layout before changing build flags. The project currently requires +CMake 3.20 and produces `build/bin/monkey`. + +## Commit style used in this repository + +Follow the existing Git history, not the older wording in `CONTRIBUTING.md`. + +### Subject format + +Use a short, lowercase, scope-prefixed subject. The common patterns are: + +- `build: bump to v1.8.7` +- `server: clean thread destroy on worker loop exit` +- `server: http: move initialization of request headers to request init` +- `core: event: Plug descriptor leaks in an error case.` +- `parser: fixed header loss issue caused by duplicated headers` +- `logger: set log file permissions to 0600, closes CVE-2013-1771 (#413)` + +### Prefix rules + +Pick the narrowest stable prefix that matches the area being changed. + +- `build:` for version bumps, CMake, workflows, packaging +- `core:` for `mk_core/` functionality +- `server:` for `mk_server/` functionality +- `server: http:` for `mk_server/mk_http.c` and closely related request flow +- `server: parser:` or `server: http_parser:` for parser-specific work +- `plugin:` or a plugin-specific prefix when the change is isolated there +- `test:` for tests +- `logger:`, `scheduler:`, `mimetype:`, `config:` when the change is clearly + isolated to that subsystem and history already uses that style +- backend-specific prefixes like `mk_event_kqueue:` are acceptable when the + change is narrow and entirely local to that backend + +### Subject style rules + +- keep it concise +- prefer lowercase after the prefix +- use colon-separated scopes, not bracket tags +- do not invent long marketing titles +- keep the subject under 80 characters +- match existing nouns already used in history where possible + +Good examples for this tree: + +- `server: http: reject malformed range delimiters` +- `server: http: avoid reusing invalid request state` +- `server: parser: validate chunk length tokens strictly` +- `core: memory: handle null mk_ptr_to_buf input` + +Bad examples for this tree: + +- `Fix CVEs` +- `Monkey: important security fixes` +- `HTTP: Add Various Improvements` +- `misc: cleanup` + +## Commit body rules + +The older contribution guide still applies well here. + +- include a body for non-trivial changes +- wrap body lines at about 80 columns +- explain the bug, the fix, and any verification done +- if the change is security-related, describe the faulty path precisely +- if multiple root causes exist, prefer separate commits + +When I am asked to commit in this repository, default behavior should be: + +1. split unrelated changes into separate commits +2. choose the narrowest prefix from the existing history +3. write a short lowercase subject +4. add a body for anything beyond trivial cleanup +5. use `git commit -s` unless the user explicitly asks otherwise + +## Working rules for this repository + +- Do not touch unrelated untracked files in the worktree. +- Be careful around parser and request lifecycle code. Many bugs surface later + in teardown, not at the first invalid input. +- Prefer minimal targeted fixes over broad refactors unless requested. +- When a bug crosses files, still group the commit by root cause, not by file. +- For security fixes, verify behavior on the built binary, not only by code + inspection. diff --git a/lib/monkey/CMakeLists.txt b/lib/monkey/CMakeLists.txt index 9797f474e41..bfb3266fc99 100644 --- a/lib/monkey/CMakeLists.txt +++ b/lib/monkey/CMakeLists.txt @@ -23,7 +23,7 @@ endif() # Monkey Version set(MK_VERSION_MAJOR 1) set(MK_VERSION_MINOR 8) -set(MK_VERSION_PATCH 6) +set(MK_VERSION_PATCH 8) set(MK_VERSION_STR "${MK_VERSION_MAJOR}.${MK_VERSION_MINOR}.${MK_VERSION_PATCH}") # Output paths diff --git a/lib/monkey/CONTRIBUTING.md b/lib/monkey/CONTRIBUTING.md index e82a8af7378..6f128a82e62 100644 --- a/lib/monkey/CONTRIBUTING.md +++ b/lib/monkey/CONTRIBUTING.md @@ -21,8 +21,10 @@ You have to pay attention to the code indentation, tabs are 4 spaces, spaces on When you commit your local changes in your repository (before to push to Github), we need you take care of the following: - - Your principal commit message (one line subject) must be prefixed with the core section name, e.g: If you are adding a new but missing protocol feature it could be __HTTP: add new XYZ method__. + - Your principal commit message (one line subject) must be prefixed with the affected area name. Follow the style used in the existing history, e.g: `build: ...`, `core: ...`, `server: ...`, `server: http: ...`, `server: parser: ...`. - The Subject of the commit must not be longer than 80 characters. + - Keep the subject short and prefer lowercase wording after the prefix. + - Use the narrowest practical scope prefix for the change. - On the commit body, each line should not be longer than 80 characters. - On most of cases we want full description about what your patch is doing, the patch description should be self descriptive.. like for dummies. Do not assume everybody knows what you are doing and on each like do not exceed 80 characters. - When running the __git commit__ command, make sure you are using the __-s__ flag, that will add a Signed-off comment in the patch description. @@ -31,19 +33,27 @@ Expanding a bit the example feature message we could use the following command: > $ git commit -a -s > -> HTTP: add new XYZ method +> server: http: add new xyz method > -> This patch adds the missing XYZ method described in RCF2616 in the +> This patch adds the missing XYZ method described in RFC2616 in the > section 12.4.x.a, it do not alter the core behavior but if the new > method is requested it will take care of the proper handling. > -> the patch have been tested using tools A & B. +> The patch has been tested using tools A & B. > > Signed-off-by: Your Name +Some recent examples from this repository are: + + - `server: clean thread destroy on worker loop exit` + - `server: http: move initialization of request headers to request init` + - `server: parser: remove unnecessary index updater` + - `build: bump to v1.8.7` + - `core: event: Plug descriptor leaks in an error case.` + If you want to see a real example, run the following command: -> $ git log 4efbc11bafeb56fbe2b4f0f6925671630ce84125 +> $ git log --oneline --no-merges -20 Your path/patches should be fully documented, that will make the review process faster for us, and a faster merge for you. diff --git a/lib/monkey/include/monkey/mk_http_parser.h b/lib/monkey/include/monkey/mk_http_parser.h index 9e3b365eef7..465ea0e4962 100644 --- a/lib/monkey/include/monkey/mk_http_parser.h +++ b/lib/monkey/include/monkey/mk_http_parser.h @@ -389,7 +389,11 @@ int mk_http_parser_chunked_decode_buf(struct mk_http_parser *p, static inline int mk_http_parser_more(struct mk_http_parser *p, int len) { - if (abs(len - p->i) - 1 > 0) { + if (len <= 0 || p->i < 0) { + return MK_FALSE; + } + + if ((p->i + 1) < len) { return MK_TRUE; } diff --git a/lib/monkey/mk_core/mk_memory.c b/lib/monkey/mk_core/mk_memory.c index c4073e23161..008f7ac6b0e 100644 --- a/lib/monkey/mk_core/mk_memory.c +++ b/lib/monkey/mk_core/mk_memory.c @@ -52,6 +52,16 @@ char *mk_ptr_to_buf(mk_ptr_t p) { char *buf; + if (!p.data || p.len == 0) { + buf = mk_mem_alloc(1); + if (!buf) { + return NULL; + } + + buf[0] = '\0'; + return buf; + } + buf = mk_mem_alloc(p.len + 1); if (!buf) return NULL; diff --git a/lib/monkey/mk_server/mk_http.c b/lib/monkey/mk_server/mk_http.c index ad12a74a045..f2f12554c0b 100644 --- a/lib/monkey/mk_server/mk_http.c +++ b/lib/monkey/mk_server/mk_http.c @@ -457,6 +457,10 @@ static int mk_http_range_parse(struct mk_http_request *sr) if ((sep_pos = mk_string_char_search(sr->range.data, '-', sr->range.len)) < 0) return -1; + if (sep_pos < eq_pos) { + return -1; + } + len = sr->range.len; sh = &sr->headers; @@ -476,10 +480,16 @@ static int mk_http_range_parse(struct mk_http_request *sr) /* =yyy-xxx */ if ((eq_pos + 1 != sep_pos) && (len > sep_pos + 1)) { buffer = mk_string_copy_substr(sr->range.data, eq_pos + 1, sep_pos); + if (!buffer) { + return -1; + } sh->ranges[0] = (unsigned long) atol(buffer); mk_mem_free(buffer); buffer = mk_string_copy_substr(sr->range.data, sep_pos + 1, len); + if (!buffer) { + return -1; + } sh->ranges[1] = (unsigned long) atol(buffer); mk_mem_free(buffer); @@ -493,6 +503,9 @@ static int mk_http_range_parse(struct mk_http_request *sr) /* =yyy- */ if ((eq_pos + 1 != sep_pos) && (len == sep_pos + 1)) { buffer = mk_string_copy_substr(sr->range.data, eq_pos + 1, len); + if (!buffer) { + return -1; + } sr->headers.ranges[0] = (unsigned long) atol(buffer); mk_mem_free(buffer); @@ -522,7 +535,16 @@ static int mk_http_directory_redirect_check(struct mk_http_session *cs, return 0; } + if (!sr->host.data || sr->host.len <= 0) { + mk_http_error(MK_CLIENT_BAD_REQUEST, cs, sr, server); + return -1; + } + host = mk_ptr_to_buf(sr->host); + if (!host) { + mk_http_error(MK_CLIENT_BAD_REQUEST, cs, sr, server); + return -1; + } /* * Add ending slash to the location string @@ -588,6 +610,9 @@ static inline char *mk_http_index_lookup(mk_ptr_t *path_base, } off = path_base->len; + if ((size_t) off >= buf_size) { + return NULL; + } memcpy(buf, path_base->data, off); mk_list_foreach(head, server->index_files) { @@ -1138,15 +1163,27 @@ int mk_http_request_end(struct mk_http_session *cs, struct mk_server *server) ret = mk_http_parser_more(&cs->parser, cs->body_length); if (ret == MK_TRUE) { /* Our pipeline request limit is the same that our keepalive limit */ + if (cs->parser.i < 0 || + (unsigned int) (cs->parser.i + 1) >= cs->body_length) { + goto shutdown; + } + cs->counter_connections++; len = (cs->body_length - cs->parser.i) -1; + if (len <= 0) { + goto shutdown; + } memmove(cs->body, cs->body + cs->parser.i + 1, len); cs->body_length = len; /* Prepare for next one */ - sr = mk_list_entry_first(&cs->request_list, struct mk_http_request, _head); + if (mk_list_is_empty(&cs->request_list) == 0) { + cs->close_now = MK_TRUE; + goto shutdown; + } + sr = &cs->sr_fixed; mk_http_request_free(sr, server); mk_http_request_init(cs, sr, server); mk_http_parser_init(&cs->parser); @@ -1626,9 +1663,10 @@ int mk_http_sched_done(struct mk_sched_conn *conn, struct mk_http_request *sr; session = mk_http_session_get(conn); - sr = mk_list_entry_first(&session->request_list, - struct mk_http_request, _head); - mk_plugin_stage_run_40(session, sr, server); + if (mk_list_is_empty(&session->request_list) != 0) { + sr = &session->sr_fixed; + mk_plugin_stage_run_40(session, sr, server); + } return mk_http_request_end(session, server); } diff --git a/lib/monkey/mk_server/mk_http_parser.c b/lib/monkey/mk_server/mk_http_parser.c index 9413528ab6e..3c831f293c1 100644 --- a/lib/monkey/mk_server/mk_http_parser.c +++ b/lib/monkey/mk_server/mk_http_parser.c @@ -173,6 +173,16 @@ static inline void request_set(mk_ptr_t *ptr, struct mk_http_parser *p, char *bu static inline int header_cmp(const char *expected, char *value, int len) { int i = 0; + size_t expected_len; + + if (len < 0) { + return -1; + } + + expected_len = strlen(expected); + if ((size_t) len != expected_len) { + return -1; + } if (len >= 8) { if (expected[0] != tolower(value[0])) return -1; @@ -535,6 +545,9 @@ static int http_parser_transfer_encoding_chunked(struct mk_http_parser *p, (errno != 0)) { return MK_HTTP_PARSER_ERROR; } + if (ptr == tmp || *ptr != '\0') { + return MK_HTTP_PARSER_ERROR; + } if (chunk_len < 0) { return MK_HTTP_PARSER_ERROR; diff --git a/lib/monkey/mk_server/mk_mimetype.c b/lib/monkey/mk_server/mk_mimetype.c index b86b4ef1a05..5462ea5c776 100644 --- a/lib/monkey/mk_server/mk_mimetype.c +++ b/lib/monkey/mk_server/mk_mimetype.c @@ -197,7 +197,12 @@ struct mk_mimetype *mk_mimetype_find(struct mk_server *server, mk_ptr_t *filenam { int j, len; - j = len = filename->len; + if (!filename->data || filename->len <= 0) { + return NULL; + } + + len = filename->len; + j = len - 1; /* looking for extension */ while (j >= 0 && filename->data[j] != '.') { diff --git a/lib/monkey/mk_server/mk_scheduler.c b/lib/monkey/mk_server/mk_scheduler.c index a680d3cdfe2..3cf0ba406e0 100644 --- a/lib/monkey/mk_server/mk_scheduler.c +++ b/lib/monkey/mk_server/mk_scheduler.c @@ -598,8 +598,10 @@ int mk_sched_check_timeouts(struct mk_sched_worker *sched, MK_TRACE("Scheduler, closing fd %i due TIMEOUT", conn->event.fd); MK_LT_SCHED(conn->event.fd, "TIMEOUT_CONN_PENDING"); - conn->protocol->cb_close(conn, sched, MK_SCHED_CONN_TIMEOUT, - server); + if (conn->protocol->cb_close) { + conn->protocol->cb_close(conn, sched, MK_SCHED_CONN_TIMEOUT, + server); + } mk_sched_drop_connection(conn, sched, server); } } @@ -749,7 +751,7 @@ int mk_sched_event_close(struct mk_sched_conn *conn, MK_TRACE("[FD %i] Connection Handler, closed", conn->event.fd); mk_event_del(sched->loop, &conn->event); - if (type != MK_EP_SOCKET_DONE) { + if (type != MK_EP_SOCKET_DONE && conn->protocol->cb_close) { conn->protocol->cb_close(conn, sched, type, server); } /* diff --git a/lib/monkey/mk_server/mk_server.c b/lib/monkey/mk_server/mk_server.c index a84ef448571..82c7a403b95 100644 --- a/lib/monkey/mk_server/mk_server.c +++ b/lib/monkey/mk_server/mk_server.c @@ -578,6 +578,10 @@ void mk_server_worker_loop(struct mk_server *server) if (timeout_fd > 0) { mk_event_timeout_destroy(evl, server_timeout); } + + mk_sched_threads_destroy_all(sched); + mk_sched_event_free_all(sched); + mk_mem_free(MK_TLS_GET(mk_tls_server_timeout)); mk_server_listen_exit(sched->listeners); mk_event_loop_destroy(evl); diff --git a/lib/monkey/mk_server/mk_user.c b/lib/monkey/mk_server/mk_user.c index 7200ff08cb6..716331acdd7 100644 --- a/lib/monkey/mk_server/mk_user.c +++ b/lib/monkey/mk_server/mk_user.c @@ -46,7 +46,7 @@ int mk_user_init(struct mk_http_session *cs, struct mk_http_request *sr, } limit = mk_string_char_search(sr->uri_processed.data + offset, '/', - sr->uri_processed.len); + sr->uri_processed.len - offset); if (limit == -1) { limit = (sr->uri_processed.len) - offset; diff --git a/packaging/build-config.json b/packaging/build-config.json index 4ca4f44df99..e6ede13a483 100644 --- a/packaging/build-config.json +++ b/packaging/build-config.json @@ -144,6 +144,14 @@ "target": "ubuntu/24.04.arm64v8", "type": "deb" }, + { + "target": "ubuntu/26.04", + "type": "deb" + }, + { + "target": "ubuntu/26.04.arm64v8", + "type": "deb" + }, { "target": "raspbian/bookworm", "type": "deb" diff --git a/packaging/distros/ubuntu/Dockerfile b/packaging/distros/ubuntu/Dockerfile index 42e1c5d2bda..b397af635e7 100644 --- a/packaging/distros/ubuntu/Dockerfile +++ b/packaging/distros/ubuntu/Dockerfile @@ -257,6 +257,56 @@ RUN apt-get update && \ ENV PATH="${CMAKE_HOME}/bin:${PATH}" +# ubuntu/26.04 base image +FROM ubuntu:26.04 AS ubuntu-26.04-base +ENV DEBIAN_FRONTEND="noninteractive" \ + CMAKE_HOME="/opt/cmake" + +ARG CMAKE_VERSION="3.31.6" +ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all libpq5 \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl3 libcurl4-openssl-dev \ + libyaml-dev pkg-config zlib1g-dev \ + tar gzip && \ + apt-get install -y --reinstall lsb-base lsb-release && \ + mkdir -p "${CMAKE_HOME}" && \ + cmake_download_url="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ + echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ + curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 + +ENV PATH="${CMAKE_HOME}/bin:${PATH}" + +# ubuntu/26.04.arm64v8 base image +FROM arm64v8/ubuntu:26.04 AS ubuntu-26.04.arm64v8-base +ENV DEBIAN_FRONTEND="noninteractive" \ + CMAKE_HOME="/opt/cmake" + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +ARG CMAKE_VERSION="3.31.6" +ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential libsystemd-dev \ + make bash wget unzip nano vim valgrind dh-make flex bison \ + libpq-dev postgresql-server-dev-all libpq5 \ + libsasl2-2 libsasl2-dev openssl libssl-dev libssl3 libcurl4-openssl-dev \ + libyaml-dev pkg-config zlib1g-dev \ + tar gzip && \ + apt-get install -y --reinstall lsb-base lsb-release && \ + mkdir -p "${CMAKE_HOME}" && \ + cmake_download_url="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ + echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ + curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 + +ENV PATH="${CMAKE_HOME}/bin:${PATH}" + # Common build for all distributions now # hadolint ignore=DL3006 FROM $BASE_BUILDER AS builder diff --git a/packaging/update-repos.sh b/packaging/update-repos.sh index aff62af7d64..be214a50797 100755 --- a/packaging/update-repos.sh +++ b/packaging/update-repos.sh @@ -65,6 +65,7 @@ DEB_REPO_PATHS=( "debian/bookworm" "debian/trixie" "ubuntu/jammy" "ubuntu/noble" + "ubuntu/resolute" "raspbian/bookworm" ) diff --git a/plugins/filter_kubernetes/kube_conf.c b/plugins/filter_kubernetes/kube_conf.c index eaf760fed4b..73e73495e92 100644 --- a/plugins/filter_kubernetes/kube_conf.c +++ b/plugins/filter_kubernetes/kube_conf.c @@ -243,11 +243,6 @@ void flb_kube_conf_destroy(struct flb_kube *ctx) if (ctx->kube_api_upstream) { flb_upstream_destroy(ctx->kube_api_upstream); } - - if (ctx->aws_pod_association_tls) { - flb_tls_destroy(ctx->aws_pod_association_tls); - } - if (ctx->aws_pod_association_upstream) { flb_upstream_destroy(ctx->aws_pod_association_upstream); } @@ -256,6 +251,10 @@ void flb_kube_conf_destroy(struct flb_kube *ctx) flb_free(ctx->platform); } + if (ctx->aws_pod_association_tls) { + flb_tls_destroy(ctx->aws_pod_association_tls); + } + #ifdef FLB_HAVE_TLS if (ctx->tls) { flb_tls_destroy(ctx->tls); diff --git a/plugins/filter_kubernetes/kube_regex.h b/plugins/filter_kubernetes/kube_regex.h index e1ea5f0863e..7f6d4a28f53 100644 --- a/plugins/filter_kubernetes/kube_regex.h +++ b/plugins/filter_kubernetes/kube_regex.h @@ -22,7 +22,7 @@ #include "kube_conf.h" -#define KUBE_TAG_TO_REGEX "(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\\.log$" +#define KUBE_TAG_TO_REGEX "(?[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\\.log$" #define KUBE_JOURNAL_TO_REGEX "^(?[^_]+)_(?[^\\._]+)(\\.(?[^_]+))?_(?[^_]+)_(?[^_]+)_[^_]+_[^_]+$" diff --git a/plugins/filter_kubernetes/kubernetes_aws.c b/plugins/filter_kubernetes/kubernetes_aws.c index 6e755353482..6173b1f76c5 100644 --- a/plugins/filter_kubernetes/kubernetes_aws.c +++ b/plugins/filter_kubernetes/kubernetes_aws.c @@ -30,6 +30,7 @@ #include "fluent-bit/flb_http_client.h" #include "fluent-bit/flb_filter_plugin.h" #include "fluent-bit/flb_pack.h" +#include "fluent-bit/flb_upstream.h" #include "fluent-bit/flb_upstream_conn.h" /* * If a file exists called service.map, load it and use it. @@ -245,6 +246,12 @@ int fetch_pod_service_map(struct flb_kube *ctx, char *api_server_url, if (!c) { flb_error("[kubernetes] could not create HTTP client"); +#ifdef FLB_HAVE_TLS + if (u_conn->tls_session != NULL) { + flb_tls_session_destroy(u_conn->tls_session); + } +#endif + flb_upstream_conn_recycle(u_conn, FLB_FALSE); flb_upstream_conn_release(u_conn); flb_upstream_destroy(ctx->aws_pod_association_upstream); flb_tls_destroy(ctx->aws_pod_association_tls); @@ -265,6 +272,12 @@ int fetch_pod_service_map(struct flb_kube *ctx, char *api_server_url, c->resp.payload); } flb_http_client_destroy(c); +#ifdef FLB_HAVE_TLS + if (u_conn->tls_session != NULL) { + flb_tls_session_destroy(u_conn->tls_session); + } +#endif + flb_upstream_conn_recycle(u_conn, FLB_FALSE); flb_upstream_conn_release(u_conn); return -1; } @@ -276,9 +289,19 @@ int fetch_pod_service_map(struct flb_kube *ctx, char *api_server_url, parse_pod_service_map(ctx, c->resp.payload, c->resp.payload_size, mutex); } - /* Cleanup */ + /* Destroy TLS session explicitly; the background thread's event loop never drains the destroy_queue. */ flb_http_client_destroy(c); +#ifdef FLB_HAVE_TLS + if (u_conn->tls_session != NULL) { + flb_tls_session_destroy(u_conn->tls_session); + } +#endif + flb_upstream_conn_recycle(u_conn, FLB_FALSE); flb_upstream_conn_release(u_conn); + flb_upstream_destroy(ctx->aws_pod_association_upstream); + flb_tls_destroy(ctx->aws_pod_association_tls); + ctx->aws_pod_association_upstream = NULL; + ctx->aws_pod_association_tls = NULL; } return 0; } diff --git a/plugins/filter_log_to_metrics/log_to_metrics.c b/plugins/filter_log_to_metrics/log_to_metrics.c index df887bfeb2d..5d0d526017e 100644 --- a/plugins/filter_log_to_metrics/log_to_metrics.c +++ b/plugins/filter_log_to_metrics/log_to_metrics.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,131 @@ static void delete_rules(struct log_to_metrics_ctx *ctx) } } +static int count_labels(struct log_to_metrics_ctx *ctx, + struct flb_filter_instance *f_ins, + int *out_k8s_count) +{ + struct mk_list *head; + struct flb_kv *kv; + int count = 0; + int k8s_count = 0; + + if (out_k8s_count == NULL) { + return -1; + } + + if (ctx->kubernetes_mode) { + k8s_count = NUMBER_OF_KUBERNETES_LABELS; + count += k8s_count; + } + + mk_list_foreach(head, &f_ins->properties) { + kv = mk_list_entry(head, struct flb_kv, _head); + + if (strcasecmp(kv->key, "label_field") == 0) { + count++; + } + else if (strcasecmp(kv->key, "add_label") == 0) { + count++; + } + } + + *out_k8s_count = k8s_count; + + if (count > MAX_LABEL_COUNT) { + flb_plg_error(ctx->ins, + "too many labels configured: %d (max=%d)", + count, MAX_LABEL_COUNT); + return -1; + } + + return count; +} + +static int prepare_label_runtime(struct log_to_metrics_ctx *ctx) +{ + int i; + int k8s_count = 0; + char fmt[MAX_LABEL_LENGTH]; + + k8s_count = ctx->kubernetes_mode ? NUMBER_OF_KUBERNETES_LABELS : 0; + + if (ctx->label_counter <= 0) { + return 0; + } + + /* Allocate pointer array for cmetrics */ + ctx->label_values = flb_calloc(ctx->label_counter, sizeof(char *)); + if (!ctx->label_values) { + flb_errno(); + return -1; + } + + /* Allocate contiguous buffer for label values */ + ctx->label_values_buf = flb_calloc(ctx->label_counter, MAX_LABEL_LENGTH); + if (!ctx->label_values_buf) { + flb_errno(); + goto error; + } + + for (i = 0; i < ctx->label_counter; i++) { + ctx->label_values[i] = ctx->label_values_buf + (i * MAX_LABEL_LENGTH); + ctx->label_values[i][0] = '\0'; + } + + /* Pre-create record accessors for each label */ + ctx->label_ras = flb_calloc(ctx->label_counter, sizeof(struct flb_record_accessor *)); + if (!ctx->label_ras) { + flb_errno(); + goto error; + } + + /* Kubernetes labels are always at the beginning (set_labels does that) */ + for (i = 0; i < k8s_count; i++) { + snprintf(fmt, sizeof(fmt) - 1, "$kubernetes['%s']", kubernetes_label_keys[i]); + ctx->label_ras[i] = flb_ra_create(fmt, FLB_TRUE); + if (!ctx->label_ras[i]) { + flb_warn("invalid record accessor key '%s' (kubernetes label)", fmt); + /* keep NULL; we will treat as missing at runtime */ + } + } + + for (i = k8s_count; i < ctx->label_counter; i++) { + ctx->label_ras[i] = flb_ra_create(ctx->label_accessors[i], FLB_TRUE); + if (!ctx->label_ras[i]) { + flb_warn("invalid record accessor key '%s' (label accessor)", ctx->label_accessors[i]); + /* keep NULL; we will treat as missing at runtime */ + } + } + + return 0; + +error: + if (ctx->label_values) { + flb_free(ctx->label_values); + ctx->label_values = NULL; + } + if (ctx->label_values_buf) { + flb_free(ctx->label_values_buf); + ctx->label_values_buf = NULL; + } + if (ctx->label_accessors) { + for (i = 0; i < ctx->label_counter; i++) { + flb_free(ctx->label_accessors[i]); + } + flb_free(ctx->label_accessors); + ctx->label_accessors = NULL; + } + if (ctx->label_ras) { + for (i = 0; i < ctx->label_counter; i++) { + flb_ra_destroy(ctx->label_ras[i]); + } + flb_free(ctx->label_ras); + ctx->label_ras = NULL; + } + return -1; +} + static int log_to_metrics_destroy(struct log_to_metrics_ctx *ctx) { int i; @@ -83,14 +209,31 @@ static int log_to_metrics_destroy(struct log_to_metrics_ctx *ctx) flb_ra_destroy(ctx->value_ra); } + /* Destroy pre-created label accessors */ + if (ctx->label_ras != NULL) { + for (i = 0; i < ctx->label_counter; i++) { + if (ctx->label_ras[i]) { + flb_ra_destroy(ctx->label_ras[i]); + } + } + flb_free(ctx->label_ras); + } + + if (ctx->label_values != NULL) { + flb_free(ctx->label_values); + } + if (ctx->label_values_buf != NULL) { + flb_free(ctx->label_values_buf); + } + if (ctx->label_accessors != NULL) { - for (i = 0; i < MAX_LABEL_COUNT; i++) { + for (i = 0; i < ctx->label_counter; i++) { flb_free(ctx->label_accessors[i]); } flb_free(ctx->label_accessors); } if (ctx->label_keys != NULL) { - for (i = 0; i < MAX_LABEL_COUNT; i++) { + for (i = 0; i < ctx->label_counter; i++) { flb_free(ctx->label_keys[i]); } flb_free(ctx->label_keys); @@ -229,27 +372,67 @@ static inline int grep_filter_data(msgpack_object map, return GREP_RET_KEEP; } +/* + * set_labels() + * Two-pass implementation: + * 1) Count labels (k8s + label_field + add_label) + * 2) Allocate exact-sized arrays and fill them + * + * Layout guarantee: + * - If kubernetes_mode enabled, indices [0..k8s_count-1] are kubernetes labels. + * - Remaining indices are user-defined labels in property iteration order. + */ static int set_labels(struct log_to_metrics_ctx *ctx, - char **label_accessors, - char **label_keys, struct flb_filter_instance *f_ins) { - struct mk_list *head; struct mk_list *split; - flb_sds_t tmp; struct flb_kv *kv; struct flb_split_entry *sentry; int counter = 0; + int total = 0; + int k8s_count = 0; int i; - if (MAX_LABEL_COUNT < NUMBER_OF_KUBERNETES_LABELS){ + total = count_labels(ctx, f_ins, &k8s_count); + if (total < 0) { return -1; } - if (ctx->kubernetes_mode){ - for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++){ - snprintf(label_keys[i], MAX_LABEL_LENGTH - 1, "%s", kubernetes_label_keys[i]); + ctx->label_counter = total; + + if (ctx->label_counter == 0) { + ctx->label_keys = NULL; + ctx->label_accessors = NULL; + return 0; + } + + ctx->label_keys = flb_calloc(ctx->label_counter, sizeof(char *)); + if (!ctx->label_keys) { + flb_errno(); + goto error; + } + + ctx->label_accessors = flb_calloc(ctx->label_counter, sizeof(char *)); + if (!ctx->label_accessors) { + flb_errno(); + goto error; + } + + if (ctx->kubernetes_mode) { + for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++) { + /* label key is the exported label name */ + ctx->label_keys[i] = flb_strdup(kubernetes_label_keys[i]); + if (!ctx->label_keys[i]) { + flb_errno(); + goto error; + } + /* + * We don't store kubernetes accessors here: + * runtime uses pre-created ctx->label_ras[0..k8s_count-1] + * built from kubernetes_label_keys[]. + */ + ctx->label_accessors[i] = NULL; } counter = NUMBER_OF_KUBERNETES_LABELS; } @@ -258,32 +441,52 @@ static int set_labels(struct log_to_metrics_ctx *ctx, mk_list_foreach(head, &f_ins->properties) { kv = mk_list_entry(head, struct flb_kv, _head); - if (counter >= MAX_LABEL_COUNT) { - return MAX_LABEL_COUNT; - } - if (strcasecmp(kv->key, "label_field") == 0) { - snprintf(label_accessors[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val); - snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", kv->val); + if (counter >= ctx->label_counter) { + flb_plg_error(ctx->ins, "internal label counter overflow"); + goto error; + } + + /* name and accessor are the same string */ + ctx->label_keys[counter] = flb_strdup(kv->val); + if (!ctx->label_keys[counter]) { + flb_errno(); + goto error; + } + ctx->label_accessors[counter] = flb_strdup(kv->val); + if (!ctx->label_accessors[counter]) { + flb_errno(); + goto error; + } counter++; } else if (strcasecmp(kv->key, "add_label") == 0) { + if (counter >= ctx->label_counter) { + flb_plg_error(ctx->ins, "internal label counter overflow"); + goto error; + } split = flb_utils_split(kv->val, ' ', 1); if (mk_list_size(split) != 2) { flb_plg_error(ctx->ins, "invalid label, expected name and key"); flb_utils_split_free(split); - return -1; + goto error; } sentry = mk_list_entry_first(split, struct flb_split_entry, _head); - tmp = flb_sds_create_len(sentry->value, sentry->len); - snprintf(label_keys[counter], MAX_LABEL_LENGTH - 1, "%s", tmp); - flb_sds_destroy(tmp); + ctx->label_keys[counter] = flb_strndup(sentry->value, sentry->len); + if (!ctx->label_keys[counter]) { + flb_errno(); + flb_utils_split_free(split); + goto error; + } sentry = mk_list_entry_last(split, struct flb_split_entry, _head); - tmp = flb_sds_create_len(sentry->value, sentry->len); - snprintf(label_accessors[counter], MAX_LABEL_LENGTH - 1, "%s", tmp); - flb_sds_destroy(tmp); + ctx->label_accessors[counter] = flb_strndup(sentry->value, sentry->len); + if (!ctx->label_accessors[counter]) { + flb_errno(); + flb_utils_split_free(split); + goto error; + } counter++; flb_utils_split_free(split); @@ -293,7 +496,32 @@ static int set_labels(struct log_to_metrics_ctx *ctx, } } - return counter; + /* Safety: counter should match computed total */ + if (counter != ctx->label_counter) { + flb_plg_error(ctx->ins, + "label count mismatch: computed=%d filled=%d", + ctx->label_counter, counter); + return -1; + } + + return ctx->label_counter; + +error: + if (ctx->label_keys) { + for (i = 0; i < ctx->label_counter; i++) { + flb_free(ctx->label_keys[i]); + } + flb_free(ctx->label_keys); + ctx->label_keys = NULL; + } + if (ctx->label_accessors) { + for (i = 0; i < ctx->label_counter; i++) { + flb_free(ctx->label_accessors[i]); + } + flb_free(ctx->label_accessors); + ctx->label_accessors = NULL; + } + return -1; } static int convert_double(char *str, double *value) @@ -389,71 +617,6 @@ static int set_buckets(struct log_to_metrics_ctx *ctx, return 0; } -static int fill_labels(struct log_to_metrics_ctx *ctx, char **label_values, - char kubernetes_label_values - [NUMBER_OF_KUBERNETES_LABELS][MAX_LABEL_LENGTH], - char **label_accessors, int label_counter, msgpack_object map) -{ - int label_iterator_start = 0; - int i; - struct flb_record_accessor *ra = NULL; - struct flb_ra_value *rval = NULL; - - if (label_counter == 0 && !ctx->kubernetes_mode){ - return 0; - } - if (MAX_LABEL_COUNT < NUMBER_OF_KUBERNETES_LABELS){ - flb_errno(); - return -1; - } - - if (ctx->kubernetes_mode){ - for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++){ - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%s", kubernetes_label_values[i]); - } - label_iterator_start = NUMBER_OF_KUBERNETES_LABELS; - } - - for (i = label_iterator_start; i < label_counter; i++){ - ra = flb_ra_create(label_accessors[i], FLB_TRUE); - if (!ra) { - flb_warn("invalid record accessor key, aborting"); - break; - } - - rval = flb_ra_get_value_object(ra, map); - if (!rval) { - /* Set value to empty string, so the value will be dropped in Cmetrics*/ - label_values[i][0] = '\0'; - } - else if (rval->type == FLB_RA_STRING) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%s", - rval->val.string); - } - else if (rval->type == FLB_RA_FLOAT) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%f", - rval->val.f64); - } - else if (rval->type == FLB_RA_INT) { - snprintf(label_values[i], MAX_LABEL_LENGTH - 1, "%ld", - (long)rval->val.i64); - } - else { - flb_warn("cannot convert given value to metric"); - break; - } - if (rval){ - flb_ra_key_value_destroy(rval); - rval = NULL; - } - if (ra){ - flb_ra_destroy(ra); - ra = NULL; - } - } - return label_counter; -} - /* Timer callback to inject metrics into the pipeline */ static void cb_send_metric_chunk(struct flb_config *config, void *data) { @@ -464,7 +627,7 @@ static void cb_send_metric_chunk(struct flb_config *config, void *data) if (ctx->cmt == NULL || ctx->input_ins == NULL) { return; } - + if (ctx->new_data) { ret = flb_input_metrics_append(ctx->input_ins, ctx->tag, strlen(ctx->tag), ctx->cmt); @@ -486,9 +649,9 @@ static void cb_send_metric_chunk(struct flb_config *config, void *data) static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, struct flb_config *config, void *data) { + int i; int ret; struct log_to_metrics_ctx *ctx; - flb_sds_t tmp; char metric_description[MAX_METRIC_LENGTH]; char metric_name[MAX_METRIC_LENGTH]; char metric_namespace[MAX_METRIC_LENGTH]; @@ -496,9 +659,11 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, char value_field[MAX_METRIC_LENGTH]; struct flb_input_instance *input_ins; struct flb_sched *sched; + const char *emitter_alias = NULL; + flb_sds_t emitter_alias_tmp = NULL; + flb_sds_t tmp = NULL; + const char *fname; - - int i; /* Create context */ ctx = flb_calloc(1, sizeof(struct log_to_metrics_ctx)); if (!ctx) { @@ -506,31 +671,28 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, return -1; } ctx->ins = f_ins; + mk_list_init(&ctx->rules); + + /* Set the context */ + flb_filter_set_context(f_ins, ctx); if (flb_filter_config_map_set(f_ins, ctx) < 0) { flb_errno(); flb_plg_error(f_ins, "configuration error"); - flb_free(ctx); return -1; } - mk_list_init(&ctx->rules); if (ctx->metric_name == NULL) { flb_plg_error(f_ins, "metric_name is not set"); - log_to_metrics_destroy(ctx); return -1; } /* Load rules */ ret = set_rules(ctx, f_ins); if (ret == -1) { - flb_free(ctx); return -1; } - /* Set the context */ - flb_filter_set_context(f_ins, ctx); - /* Set buckets for histogram */ ctx->buckets = NULL; ctx->bucket_counter = 0; @@ -538,49 +700,26 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, if (set_buckets(ctx, f_ins) < 0) { flb_plg_error(f_ins, "Setting buckets failed"); - log_to_metrics_destroy(ctx); return -1; } - ctx->label_accessors = NULL; - ctx->label_accessors = (char **) flb_calloc(1, MAX_LABEL_COUNT * sizeof(char *)); - if (!ctx->label_accessors) { - flb_errno(); - log_to_metrics_destroy(ctx); + /* Set label keys/accessors dynamically based on configured properties */ + /* ctx->label_counter is set inside set_labels() */ + ret = set_labels(ctx, f_ins); + if (ret < 0){ return -1; } - for (i = 0; i < MAX_LABEL_COUNT; i++) { - ctx->label_accessors[i] = flb_calloc(1, MAX_LABEL_LENGTH * sizeof(char)); - if (!ctx->label_accessors[i]) { - flb_errno(); - log_to_metrics_destroy(ctx); - return -1; - } - } - - /* Set label keys */ - ctx->label_keys = (char **) flb_calloc(1, MAX_LABEL_COUNT * sizeof(char *)); - for (i = 0; i < MAX_LABEL_COUNT; i++) { - ctx->label_keys[i] = flb_calloc(1, MAX_LABEL_LENGTH * sizeof(char)); - if (!ctx->label_keys[i]) { - flb_errno(); - log_to_metrics_destroy(ctx); - return -1; - } - } - - ret = set_labels(ctx, ctx->label_accessors, ctx->label_keys, f_ins); - if (ret < 0){ + /* prepare label runtime buffers + pre-create record accessors */ + if (prepare_label_runtime(ctx) < 0) { + flb_plg_error(f_ins, "failed to prepare label runtime buffers"); log_to_metrics_destroy(ctx); return -1; } - ctx->label_counter = ret; /* Check metric tag */ if (ctx->tag == NULL || strlen(ctx->tag) == 0) { flb_plg_error(f_ins, "Metric tag is not set"); - log_to_metrics_destroy(ctx); return -1; } @@ -604,13 +743,11 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, "invalid 'mode' value. Only " "'counter', 'gauge' or " "'histogram' types are allowed"); - log_to_metrics_destroy(ctx); return -1; } } else { flb_plg_error(f_ins, "configuration property not set"); - log_to_metrics_destroy(ctx); return -1; } @@ -637,7 +774,6 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, if (ctx->metric_description == NULL || strlen(ctx->metric_description) == 0) { flb_plg_error(f_ins, "metric_description is not set"); - log_to_metrics_destroy(ctx); return -1; } snprintf(metric_description, sizeof(metric_description) - 1, "%s", @@ -647,7 +783,6 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, if (ctx->mode > 0) { if (ctx->value_field == NULL || strlen(ctx->value_field) == 0) { flb_plg_error(f_ins, "value_field is not set"); - log_to_metrics_destroy(ctx); return -1; } snprintf(value_field, sizeof(value_field) - 1, "%s", @@ -656,7 +791,6 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, ctx->value_ra = flb_ra_create(ctx->value_field, FLB_TRUE); if (ctx->value_ra == NULL) { flb_plg_error(f_ins, "invalid record accessor key for value_field"); - log_to_metrics_destroy(ctx); return -1; } } @@ -671,7 +805,7 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, "0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0"); ctx->histogram_buckets = cmt_histogram_buckets_default_create(); } - else{ + else { ctx->histogram_buckets = cmt_histogram_buckets_create_size(ctx->buckets, ctx->bucket_counter); } } @@ -701,53 +835,72 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, break; default: flb_plg_error(f_ins, "unsupported mode"); - log_to_metrics_destroy(ctx); return -1; } - tmp = (char *) flb_filter_get_property("emitter_name", f_ins); - /* If emitter_name is not set, use the default name */ - if (tmp == NULL) { - tmp = (char *) flb_filter_name(f_ins); - ctx->emitter_name = flb_sds_create_size(64); - ctx->emitter_name = flb_sds_printf(&ctx->emitter_name, "emitter_for_%s", tmp); + if (ctx->emitter_name != NULL && flb_sds_len(ctx->emitter_name) > 0) { + emitter_alias = ctx->emitter_name; } else { - ctx->emitter_name = flb_sds_create(tmp); + fname = (const char *) flb_filter_name(f_ins); + emitter_alias_tmp = flb_sds_create_size(64); + if (!emitter_alias_tmp) { + flb_errno(); + log_to_metrics_destroy(ctx); + return -1; + } + tmp = flb_sds_printf(&emitter_alias_tmp, "emitter_for_%s", fname); + if (!tmp) { + flb_sds_destroy(emitter_alias_tmp); + emitter_alias_tmp = NULL; + flb_errno(); + log_to_metrics_destroy(ctx); + return -1; + } + emitter_alias_tmp = tmp; + emitter_alias = emitter_alias_tmp; } - ret = flb_input_name_exists(ctx->emitter_name, config); + ret = flb_input_name_exists(emitter_alias, config); if (ret) { flb_plg_error(f_ins, "emitter_name '%s' already exists", - ctx->emitter_name); - flb_sds_destroy(ctx->emitter_name); + emitter_alias); + if (emitter_alias_tmp) { + flb_sds_destroy(emitter_alias_tmp); + } log_to_metrics_destroy(ctx); return -1; } input_ins = flb_input_new(config, "emitter", NULL, FLB_FALSE); if (!input_ins) { flb_plg_error(f_ins, "cannot create metrics emitter instance"); - flb_sds_destroy(ctx->emitter_name); + if (emitter_alias_tmp) { + flb_sds_destroy(emitter_alias_tmp); + } + log_to_metrics_destroy(ctx); return -1; } /* Set the alias for emitter */ - ret = flb_input_set_property(input_ins, "alias", ctx->emitter_name); + ret = flb_input_set_property(input_ins, "alias", emitter_alias); if (ret == -1) { flb_plg_warn(ctx->ins, "cannot set emitter_name"); - flb_sds_destroy(ctx->emitter_name); + if (emitter_alias_tmp) { + flb_sds_destroy(emitter_alias_tmp); + } log_to_metrics_destroy(ctx); return -1; } - flb_sds_destroy(ctx->emitter_name); + if (emitter_alias_tmp) { + flb_sds_destroy(emitter_alias_tmp); + } /* Set the storage type for emitter */ ret = flb_input_set_property(input_ins, "storage.type", "memory"); if (ret == -1) { flb_plg_error(f_ins, "cannot set storage type for emitter instance"); - log_to_metrics_destroy(ctx); return -1; } @@ -761,14 +914,12 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, if (ret == -1) { flb_errno(); flb_plg_error(f_ins, "cannot initialize metrics emitter instance."); - log_to_metrics_destroy(ctx); return -1; } ret = flb_storage_input_create(config->cio, input_ins); if (ret == -1) { flb_plg_error(ctx->ins, "cannot initialize storage for metrics stream"); - log_to_metrics_destroy(ctx); return -1; } ctx->input_ins = input_ins; @@ -786,27 +937,25 @@ static int cb_log_to_metrics_init(struct flb_filter_instance *f_ins, ctx->timer_mode = FLB_FALSE; return 0; } - + /* Initialize timer for scheduled metric updates */ sched = flb_sched_ctx_get(); if(sched == 0) { flb_plg_error(f_ins, "could not get scheduler context"); - log_to_metrics_destroy(ctx); return -1; } /* Convert flush_interval_sec and flush_interval_nsec to milliseconds */ - ctx->timer_interval = (ctx->flush_interval_sec * 1000) + + ctx->timer_interval = (ctx->flush_interval_sec * 1000) + (ctx->flush_interval_nsec / 1000000); flb_plg_debug(ctx->ins, "Creating metric timer with frequency %d ms", ctx->timer_interval); - + ret = flb_sched_timer_cb_create(sched, FLB_SCHED_TIMER_CB_PERM, ctx->timer_interval, cb_send_metric_chunk, ctx, &ctx->timer); if (ret < 0) { flb_plg_error(f_ins, "could not create timer callback"); - log_to_metrics_destroy(ctx); return -1; } ctx->timer_mode = FLB_TRUE; @@ -831,15 +980,10 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, uint64_t ts; struct log_to_metrics_ctx *ctx = context; struct flb_ra_value *rval = NULL; - struct flb_record_accessor *ra = NULL; - char fmt[MAX_LABEL_LENGTH]; - char **label_values = NULL; - int label_count = 0; int i; double gauge_value = 0; double histogram_value = 0; - char kubernetes_label_values - [NUMBER_OF_KUBERNETES_LABELS][MAX_LABEL_LENGTH]; + int label_count = ctx->label_counter; /* Create temporary msgpack buffer */ msgpack_sbuffer_init(&tmp_sbuf); @@ -860,52 +1004,42 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, ret = grep_filter_data(map, context); if (ret == GREP_RET_KEEP) { ts = cfl_time_now(); - if(ctx->kubernetes_mode) { - for (i = 0; i < NUMBER_OF_KUBERNETES_LABELS; i++) { - snprintf(fmt, MAX_LABEL_LENGTH - 1, "$kubernetes['%s']", - kubernetes_label_keys[i]); - ra = flb_ra_create(fmt, FLB_TRUE); - if (!ra) { - flb_plg_error(ctx->ins, "invalid record accessor key, aborting"); - break; + + /* Fill label values using pre-created record accessors. + * Missing/invalid -> empty string (keeps existing behavior). + */ + if (label_count > 0) { + for (i = 0; i < label_count; i++) { + ctx->label_values[i][0] = '\0'; + + if (ctx->label_ras[i] == NULL) { + continue; } - rval = flb_ra_get_value_object(ra, map); + + rval = flb_ra_get_value_object(ctx->label_ras[i], map); if (!rval) { - flb_plg_error(ctx->ins, "given value field is empty or not " - "existent: %s. Skipping labels.", fmt); - ctx->label_counter = 0; + /* keep empty string */ + continue; } - else if (rval->type != FLB_RA_STRING) { - flb_plg_error(ctx->ins, "cannot access label %s", kubernetes_label_keys[i]); - break; + + if (rval->type == FLB_RA_STRING) { + snprintf(ctx->label_values[i], MAX_LABEL_LENGTH - 1, "%s", + rval->val.string); } - else { - snprintf(kubernetes_label_values[i], - MAX_LABEL_LENGTH - 1, "%s", rval->val.string); + else if (rval->type == FLB_RA_FLOAT) { + snprintf(ctx->label_values[i], MAX_LABEL_LENGTH - 1, "%f", + rval->val.f64); } - if (rval){ - flb_ra_key_value_destroy(rval); - rval = NULL; + else if (rval->type == FLB_RA_INT) { + snprintf(ctx->label_values[i], MAX_LABEL_LENGTH - 1, "%ld", + (long) rval->val.i64); } - if (ra){ - flb_ra_destroy(ra); - ra = NULL; + else { + /* unsupported type: keep empty string */ } - } - } - if (ctx->label_counter > 0){ - /* Fill optional labels */ - label_values = flb_malloc(MAX_LABEL_COUNT * sizeof(char *)); - for (i = 0; i < MAX_LABEL_COUNT; i++) { - label_values[i] = flb_malloc(MAX_LABEL_LENGTH * - sizeof(char)); - } - label_count = fill_labels(ctx, label_values, - kubernetes_label_values, ctx->label_accessors, - ctx->label_counter, map); - if (label_count != ctx->label_counter){ - label_count = 0; + flb_ra_key_value_destroy(rval); + rval = NULL; } } @@ -913,7 +1047,7 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, switch (ctx->mode) { case FLB_LOG_TO_METRICS_COUNTER: ret = cmt_counter_inc(ctx->c, ts, label_count, - label_values); + ctx->label_values); break; case FLB_LOG_TO_METRICS_GAUGE: @@ -941,7 +1075,7 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, } ret = cmt_gauge_set(ctx->g, ts, gauge_value, - label_count, label_values); + label_count, ctx->label_values); flb_ra_key_value_destroy(rval); rval = NULL; break; @@ -971,7 +1105,7 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, } ret = cmt_histogram_observe(ctx->h, ts, histogram_value, - label_count, label_values); + label_count, ctx->label_values); flb_ra_key_value_destroy(rval); rval = NULL; break; @@ -993,17 +1127,6 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, else { ctx->new_data = FLB_TRUE; } - - /* Cleanup */ - msgpack_unpacked_destroy(&result); - if (label_values != NULL){ - for (i = 0; i < MAX_LABEL_COUNT; i++) { - if (label_values[i] != NULL){ - flb_free(label_values[i]); - } - } - flb_free(label_values); - } } else if (ret == GREP_RET_EXCLUDE) { /* Do nothing */ @@ -1028,6 +1151,11 @@ static int cb_log_to_metrics_filter(const void *data, size_t bytes, static int cb_log_to_metrics_exit(void *data, struct flb_config *config) { struct log_to_metrics_ctx *ctx = data; + + if (!ctx) { + return 0; + } + if(ctx->timer != NULL) { flb_plg_debug(ctx->ins, "Destroying callback timer"); flb_sched_timer_destroy(ctx->timer); diff --git a/plugins/filter_log_to_metrics/log_to_metrics.h b/plugins/filter_log_to_metrics/log_to_metrics.h index 9c2508a9138..7f2a1407463 100644 --- a/plugins/filter_log_to_metrics/log_to_metrics.h +++ b/plugins/filter_log_to_metrics/log_to_metrics.h @@ -25,6 +25,8 @@ #include #include +#include + /* rule types */ #define GREP_REGEX 1 #define GREP_EXCLUDE 2 @@ -46,7 +48,7 @@ #define NUMBER_OF_KUBERNETES_LABELS 5 #define MAX_LABEL_LENGTH 253 #define MAX_METRIC_LENGTH 253 -#define MAX_LABEL_COUNT 32 +#define MAX_LABEL_COUNT 128 #define FLB_MEM_BUF_LIMIT_DEFAULT "10M" #define DEFAULT_LOG_TO_METRICS_NAMESPACE "log_metric" @@ -66,6 +68,14 @@ struct log_to_metrics_ctx { int bucket_counter; double *buckets; + + /* Pre-created record accessors for each label (including kubernetes labels) */ + struct flb_record_accessor **label_ras; + + /* Pre-allocated label value buffers, reused per record */ + char *label_values_buf; /* contiguous buffer: label_counter * MAX_LABEL_LENGTH */ + char **label_values; /* pointer view: label_values[i] points into label_values_buf */ + struct cmt_counter *c; struct cmt_gauge *g; struct cmt_histogram *h; diff --git a/plugins/in_collectd/netprot.c b/plugins/in_collectd/netprot.c index a7254620d17..d6721df606b 100644 --- a/plugins/in_collectd/netprot.c +++ b/plugins/in_collectd/netprot.c @@ -247,8 +247,9 @@ int netprot_to_msgpack(char *buf, int len, struct mk_list *tdb, part_type = be16read(buf); part_len = be16read((unsigned char *) buf + 2); - if (len < part_len) { - flb_error("[in_collectd] data truncated (%i < %i)", len, part_len); + if (part_len < 4 || len < part_len) { + flb_error("[in_collectd] invalid part length (%i, remaining %i)", + part_len, len); return -1; } @@ -268,7 +269,7 @@ int netprot_to_msgpack(char *buf, int len, struct mk_list *tdb, switch (part_type) { case PART_HOST: - if (ptr[size] == '\0') { + if (size > 0 && ptr[size - 1] == '\0') { hdr.host = ptr; } break; @@ -279,22 +280,22 @@ int netprot_to_msgpack(char *buf, int len, struct mk_list *tdb, hdr.time = hr2time(be64read(ptr)); break; case PART_PLUGIN: - if (ptr[size] == '\0') { + if (size > 0 && ptr[size - 1] == '\0') { hdr.plugin = ptr; } break; case PART_PLUGIN_INSTANCE: - if (ptr[size] == '\0') { + if (size > 0 && ptr[size - 1] == '\0') { hdr.plugin_instance = ptr; } break; case PART_TYPE: - if (ptr[size] == '\0') { + if (size > 0 && ptr[size - 1] == '\0') { hdr.type = ptr; } break; case PART_TYPE_INSTANCE: - if (ptr[size] == '\0') { + if (size > 0 && ptr[size - 1] == '\0') { hdr.type_instance = ptr; } break; diff --git a/plugins/in_dummy/in_dummy.c b/plugins/in_dummy/in_dummy.c index 13270017f00..f578c09cfae 100644 --- a/plugins/in_dummy/in_dummy.c +++ b/plugins/in_dummy/in_dummy.c @@ -353,10 +353,6 @@ static int in_dummy_init(struct flb_input_instance *in, flb_input_set_context(in, ctx); - if (ctx->flush_on_startup) { - in_dummy_collect(in, config, ctx); - } - ret = flb_input_set_collector_time(in, in_dummy_collect, tm.tv_sec, @@ -374,6 +370,20 @@ static int in_dummy_init(struct flb_input_instance *in, return 0; } +static int in_dummy_pre_run(struct flb_input_instance *in, + struct flb_config *config, void *in_context) +{ + struct flb_dummy *ctx; + + ctx = (struct flb_dummy *) in_context; + + if (ctx != NULL && ctx->flush_on_startup) { + in_dummy_collect(in, config, in_context); + } + + return 0; +} + static void in_dummy_pause(void *data, struct flb_config *config) { struct flb_dummy *ctx = data; @@ -475,7 +485,7 @@ struct flb_input_plugin in_dummy_plugin = { .name = "dummy", .description = "Generate dummy data", .cb_init = in_dummy_init, - .cb_pre_run = NULL, + .cb_pre_run = in_dummy_pre_run, .cb_collect = in_dummy_collect, .cb_flush_buf = NULL, .config_map = config_map, diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c index 763cf9993e8..26e183b3d0d 100644 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c +++ b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c @@ -283,6 +283,10 @@ static int get_write_op(struct flb_in_elasticsearch *ctx, msgpack_object *map, f msgpack_object key; int check = FLB_FALSE; + if (map->via.map.size == 0) { + return FLB_FALSE; + } + kv = map->via.map.ptr; key = kv[0].key; if (key.type == MSGPACK_OBJECT_BIN) { @@ -326,7 +330,7 @@ static int process_ndpack(struct flb_in_elasticsearch *ctx, flb_sds_t tag, char msgpack_object *obj; flb_sds_t tag_from_record = NULL; int idx = 0; - flb_sds_t write_op; + flb_sds_t write_op = NULL; size_t op_str_size = 0; int op_ret = FLB_FALSE; int error_op = FLB_FALSE; @@ -501,11 +505,13 @@ static int process_ndpack(struct flb_in_elasticsearch *ctx, flb_sds_t tag, char } if (status_buffer_avail(ctx, bulk_statuses, 50) == FLB_FALSE) { flb_sds_destroy(write_op); + write_op = NULL; break; } } flb_sds_destroy(write_op); + write_op = NULL; } proceed: @@ -522,7 +528,7 @@ static int process_ndpack(struct flb_in_elasticsearch *ctx, flb_sds_t tag, char if (idx % 2 != 0) { flb_plg_warn(ctx->ins, "decode payload of Bulk API is failed"); msgpack_unpacked_destroy(&result); - if (error_op == FLB_FALSE) { + if (error_op == FLB_FALSE && write_op != NULL) { /* On lacking of body case in non-error case, there is no * releasing memory code paths. We should proceed to do * it here. */ diff --git a/plugins/in_forward/fw_prot.c b/plugins/in_forward/fw_prot.c index 2ae5488ed0b..f11e2a7f780 100644 --- a/plugins/in_forward/fw_prot.c +++ b/plugins/in_forward/fw_prot.c @@ -1089,21 +1089,22 @@ static size_t receiver_recv(struct fw_conn *conn, char *buf, size_t try_size) { return actual_size; } -static size_t receiver_to_unpacker(struct fw_conn *conn, size_t request_size, - msgpack_unpacker *unpacker) +static int receiver_to_unpacker(struct fw_conn *conn, size_t request_size, + msgpack_unpacker *unpacker, size_t *recv_len) { - size_t recv_len; + *recv_len = 0; /* make sure there's enough room, or expand the unpacker accordingly */ if (msgpack_unpacker_buffer_capacity(unpacker) < request_size) { - msgpack_unpacker_reserve_buffer(unpacker, request_size); - assert(msgpack_unpacker_buffer_capacity(unpacker) >= request_size); + if (!msgpack_unpacker_reserve_buffer(unpacker, request_size)) { + return -1; + } } - recv_len = receiver_recv(conn, msgpack_unpacker_buffer(unpacker), - request_size); - msgpack_unpacker_buffer_consumed(unpacker, recv_len); + *recv_len = receiver_recv(conn, msgpack_unpacker_buffer(unpacker), + request_size); + msgpack_unpacker_buffer_consumed(unpacker, *recv_len); - return recv_len; + return 0; } static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, @@ -1284,11 +1285,26 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) } unp = msgpack_unpacker_new(1024); + if (!unp) { + flb_plg_error(ctx->ins, "could not allocate msgpack unpacker"); + flb_sds_destroy(out_tag); + return -1; + } + msgpack_unpacked_init(&result); conn->rest = conn->buf_len; while (1) { - recv_len = receiver_to_unpacker(conn, EACH_RECV_SIZE, unp); + ret = receiver_to_unpacker(conn, EACH_RECV_SIZE, unp, &recv_len); + if (ret == -1) { + flb_plg_error(ctx->ins, "could not allocate msgpack unpacker buffer"); + msgpack_unpacked_destroy(&result); + msgpack_unpacker_free(unp); + flb_sds_destroy(out_tag); + + return -1; + } + if (recv_len == 0) { /* No more data */ msgpack_unpacker_free(unp); @@ -1462,6 +1478,14 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn) /* * Forward format 2 (message mode) : [tag, time, map, ...] */ + if (root.via.array.size < 3) { + flb_plg_warn(ctx->ins, + "message mode requires at least 3 elements"); + msgpack_unpacked_destroy(&result); + msgpack_unpacker_free(unp); + flb_sds_destroy(out_tag); + return -1; + } map = root.via.array.ptr[2]; if (map.type != MSGPACK_OBJECT_MAP) { flb_plg_warn(ctx->ins, "invalid data format, map expected"); diff --git a/plugins/in_mqtt/mqtt_prot.c b/plugins/in_mqtt/mqtt_prot.c index 69be0d88e53..5b5290fa8f9 100644 --- a/plugins/in_mqtt/mqtt_prot.c +++ b/plugins/in_mqtt/mqtt_prot.c @@ -397,7 +397,7 @@ int mqtt_prot_parser(struct mqtt_conn *conn) return MQTT_ERROR; } - if (length + 2 > (conn->buf_len - pos)) { + if (length + (conn->buf_pos - pos + 1) > (conn->buf_len - pos)) { conn->buf_pos = pos; flb_plg_trace(ctx->ins, "[fd=%i] Need more data", conn->connection->fd); @@ -405,7 +405,7 @@ int mqtt_prot_parser(struct mqtt_conn *conn) } if ((BUFC() & 128) == 0) { - if (conn->buf_len - 2 < length) { + if (conn->buf_len - (conn->buf_pos - pos + 1) < length) { conn->buf_pos = pos; flb_plg_trace(ctx->ins, "[fd=%i] Need more data", conn->connection->fd); diff --git a/plugins/in_node_exporter_metrics/ne_cpu_linux.c b/plugins/in_node_exporter_metrics/ne_cpu_linux.c index 44f8ce14293..bc7c8a9feec 100644 --- a/plugins/in_node_exporter_metrics/ne_cpu_linux.c +++ b/plugins/in_node_exporter_metrics/ne_cpu_linux.c @@ -112,7 +112,7 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) entry = mk_list_entry(head, struct flb_slist_entry, _head); /* Core ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "topology", "core_id", &core_id); @@ -121,7 +121,7 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) } /* Physical ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "topology", "physical_package_id", &physical_package_id); @@ -130,7 +130,7 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) } /* Package Metric: node_cpu_core_throttles_total */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "thermal_throttle", "core_throttle_count", &core_throttle_count); @@ -150,7 +150,7 @@ static int cpu_thermal_update(struct flb_ne *ctx, uint64_t ts) } /* Package Metric: node_cpu_package_throttles_total */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "thermal_throttle", "package_throttle_count", &package_throttle_count); @@ -307,7 +307,7 @@ static int cpu_stat_update(struct flb_ne *ctx, uint64_t ts) struct flb_slist_entry *line; struct cpu_stat_info st = {0}; - ret = ne_utils_file_read_lines(ctx->path_procfs, "/stat", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/stat", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c b/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c index 1df0862b889..40854840f61 100644 --- a/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c +++ b/plugins/in_node_exporter_metrics/ne_cpufreq_linux.c @@ -118,7 +118,7 @@ static int cpufreq_update(struct flb_ne *ctx) cpu_id++; /* node_cpu_frequency_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "cpuinfo_cur_freq", &val); if (ret == 0) { @@ -128,7 +128,7 @@ static int cpufreq_update(struct flb_ne *ctx) } /* node_cpu_frequency_max_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "cpuinfo_max_freq", &val); if (ret == 0) { @@ -138,7 +138,7 @@ static int cpufreq_update(struct flb_ne *ctx) } /* node_cpu_frequency_min_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "cpuinfo_min_freq", &val); if (ret == 0) { @@ -149,7 +149,7 @@ static int cpufreq_update(struct flb_ne *ctx) /* node_cpu_scaling_frequency_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "scaling_cur_freq", &val); if (ret == 0) { @@ -159,7 +159,7 @@ static int cpufreq_update(struct flb_ne *ctx) } /* node_cpu_scaling_frequency_max_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "scaling_max_freq", &val); if (ret == 0) { @@ -169,7 +169,7 @@ static int cpufreq_update(struct flb_ne *ctx) } /* node_cpu_frequency_min_hertz */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cpufreq", "scaling_min_freq", &val); if (ret == 0) { diff --git a/plugins/in_node_exporter_metrics/ne_diskstats_linux.c b/plugins/in_node_exporter_metrics/ne_diskstats_linux.c index 20fa230d6b4..f004bf7360b 100644 --- a/plugins/in_node_exporter_metrics/ne_diskstats_linux.c +++ b/plugins/in_node_exporter_metrics/ne_diskstats_linux.c @@ -404,7 +404,7 @@ static int diskstats_update(struct flb_ne *ctx) mk_list_init(&list); mk_list_init(&split_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/diskstats", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/diskstats", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_filefd_linux.c b/plugins/in_node_exporter_metrics/ne_filefd_linux.c index 6c89a939ef2..49a800a3e0b 100644 --- a/plugins/in_node_exporter_metrics/ne_filefd_linux.c +++ b/plugins/in_node_exporter_metrics/ne_filefd_linux.c @@ -58,7 +58,7 @@ static int filefd_update(struct flb_ne *ctx) struct flb_slist_entry *max; mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/sys/fs/file-nr", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/sys/fs/file-nr", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_hwmon_linux.c b/plugins/in_node_exporter_metrics/ne_hwmon_linux.c index 774fcae3057..dbd240b71cb 100644 --- a/plugins/in_node_exporter_metrics/ne_hwmon_linux.c +++ b/plugins/in_node_exporter_metrics/ne_hwmon_linux.c @@ -185,14 +185,14 @@ static void hwmon_process_sensor(struct flb_ne *ctx, const char *chip_path, } /* read input value */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, sensor_path, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, sensor_path, NULL, NULL, &val); if (ret != 0) { return; } snprintf(label_name, sizeof(label_name) - 1, "%s_label", sensor_name); - if (ne_utils_file_read_sds(ctx->path_sysfs, chip_path, + if (ne_utils_file_read_sds(ctx, ctx->path_sysfs, chip_path, label_name, NULL, &label) == 0) { sensor_label = label; } @@ -206,7 +206,7 @@ static void hwmon_process_sensor(struct flb_ne *ctx, const char *chip_path, 2, (char *[]) {(char *) chip_name, (char *) sensor_label}); snprintf(file_tmp, sizeof(file_tmp) - 1, "%s_max", sensor_name); - if (ne_utils_file_read_uint64(ctx->path_sysfs, chip_path, + if (ne_utils_file_read_uint64(ctx, ctx->path_sysfs, chip_path, file_tmp, NULL, &val) == 0) { cmt_gauge_set(ctx->hwmon_temp_max_celsius, tstamp, ((double) val) / 1000.0, @@ -214,7 +214,7 @@ static void hwmon_process_sensor(struct flb_ne *ctx, const char *chip_path, } snprintf(file_tmp, sizeof(file_tmp) - 1, "%s_crit", sensor_name); - if (ne_utils_file_read_uint64(ctx->path_sysfs, chip_path, + if (ne_utils_file_read_uint64(ctx, ctx->path_sysfs, chip_path, file_tmp, NULL, &val) == 0) { cmt_gauge_set(ctx->hwmon_temp_crit_celsius, tstamp, ((double) val) / 1000.0, @@ -269,7 +269,7 @@ static int ne_hwmon_update(struct flb_input_instance *ins, mk_list_foreach(head, &hwmons) { entry = mk_list_entry(head, struct flb_slist_entry, _head); - if (ne_utils_file_read_sds(ctx->path_sysfs, entry->str, + if (ne_utils_file_read_sds(ctx, ctx->path_sysfs, entry->str, "name", NULL, &chip) != 0) { continue; } diff --git a/plugins/in_node_exporter_metrics/ne_loadavg_linux.c b/plugins/in_node_exporter_metrics/ne_loadavg_linux.c index 1f7bc873cf0..7a003ea830a 100644 --- a/plugins/in_node_exporter_metrics/ne_loadavg_linux.c +++ b/plugins/in_node_exporter_metrics/ne_loadavg_linux.c @@ -67,7 +67,7 @@ static int loadavg_update(struct flb_ne *ctx) mk_list_init(&list); mk_list_init(&split_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/loadavg", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/loadavg", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_meminfo_linux.c b/plugins/in_node_exporter_metrics/ne_meminfo_linux.c index ec8a28e193a..b1f2a0e80e7 100644 --- a/plugins/in_node_exporter_metrics/ne_meminfo_linux.c +++ b/plugins/in_node_exporter_metrics/ne_meminfo_linux.c @@ -53,7 +53,7 @@ static int meminfo_configure(struct flb_ne *ctx) mk_list_init(&list); mk_list_init(&split_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/meminfo", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/meminfo", &list); if (ret == -1) { return -1; } @@ -202,7 +202,7 @@ static int meminfo_update(struct flb_ne *ctx) struct flb_slist_entry *entry; mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/meminfo", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/meminfo", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_netdev_linux.c b/plugins/in_node_exporter_metrics/ne_netdev_linux.c index baa92d2abe7..f848834ffce 100644 --- a/plugins/in_node_exporter_metrics/ne_netdev_linux.c +++ b/plugins/in_node_exporter_metrics/ne_netdev_linux.c @@ -98,7 +98,7 @@ static int netdev_configure(struct flb_ne *ctx) mk_list_init(&rx_list); mk_list_init(&tx_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/dev", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/dev", &list); if (ret == -1) { return -1; } @@ -236,7 +236,7 @@ static int netdev_update(struct flb_ne *ctx) mk_list_init(&rx_list); mk_list_init(&tx_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/dev", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/dev", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_netstat_linux.c b/plugins/in_node_exporter_metrics/ne_netstat_linux.c index 826de260b7e..4bd51b88ebf 100644 --- a/plugins/in_node_exporter_metrics/ne_netstat_linux.c +++ b/plugins/in_node_exporter_metrics/ne_netstat_linux.c @@ -215,7 +215,7 @@ static int netstat_update(struct flb_ne *ctx) int prev_proto; mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/snmp", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/snmp", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_node_exporter_metrics/ne_nvme_linux.c b/plugins/in_node_exporter_metrics/ne_nvme_linux.c index 0f5b67b86a7..4d2f88b24a7 100644 --- a/plugins/in_node_exporter_metrics/ne_nvme_linux.c +++ b/plugins/in_node_exporter_metrics/ne_nvme_linux.c @@ -105,7 +105,7 @@ static int nvme_get_entry_value(struct flb_ne *ctx, if (check_path_for_sysfs(ctx, nvme_info->str, entry_path) != 0) { return -1; } - ret = ne_utils_file_read_lines(nvme_info->str, nvme_sysentry, out_info_list); + ret = ne_utils_file_read_lines(ctx, nvme_info->str, nvme_sysentry, out_info_list); if (ret == -1) { return ret; } diff --git a/plugins/in_node_exporter_metrics/ne_processes_linux.c b/plugins/in_node_exporter_metrics/ne_processes_linux.c index fdbdc9aaeb2..c4b25b819e7 100644 --- a/plugins/in_node_exporter_metrics/ne_processes_linux.c +++ b/plugins/in_node_exporter_metrics/ne_processes_linux.c @@ -195,7 +195,7 @@ static int processes_thread_update(struct flb_ne *ctx, flb_sds_t pid_str, flb_sd } mk_list_init(&stat_list); - ret = ne_utils_file_read_lines(thread->str, "/stat", &stat_list); + ret = ne_utils_file_read_lines(ctx,thread->str, "/stat", &stat_list); if (ret == -1) { continue; } @@ -271,7 +271,7 @@ static int processes_update(struct flb_ne *ctx) ts = cfl_time_now(); - ret = ne_utils_file_read_uint64(ctx->path_procfs, "/sys", "kernel", "threads-max", &val); + ret = ne_utils_file_read_uint64(ctx, ctx->path_procfs, "/sys", "kernel", "threads-max", &val); if (ret == -1) { return -1; } @@ -282,7 +282,7 @@ static int processes_update(struct flb_ne *ctx) (double)val, 0, NULL); } - ret = ne_utils_file_read_uint64(ctx->path_procfs, "/sys", "kernel", "pid_max", &val); + ret = ne_utils_file_read_uint64(ctx, ctx->path_procfs, "/sys", "kernel", "pid_max", &val); if (ret == -1) { return -1; } @@ -313,7 +313,7 @@ static int processes_update(struct flb_ne *ctx) } mk_list_init(&stat_list); - ret = ne_utils_file_read_lines(process->str, "/stat", &stat_list); + ret = ne_utils_file_read_lines(ctx,process->str, "/stat", &stat_list); if (ret == -1) { continue; } diff --git a/plugins/in_node_exporter_metrics/ne_sockstat_linux.c b/plugins/in_node_exporter_metrics/ne_sockstat_linux.c index 6ecea879d60..4cb1b9799cc 100644 --- a/plugins/in_node_exporter_metrics/ne_sockstat_linux.c +++ b/plugins/in_node_exporter_metrics/ne_sockstat_linux.c @@ -170,7 +170,7 @@ static int sockstat_update(struct flb_ne *ctx) struct flb_slist_entry *val; mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/sockstat", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/sockstat", &list); if (ret == -1) { return -1; } @@ -298,7 +298,7 @@ static int sockstat_update(struct flb_ne *ctx) /* Parse IPv6 statistics */ mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/sockstat6", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/net/sockstat6", &list); if (ret != -1) { mk_list_foreach(head, &list) { line = mk_list_entry(head, struct flb_slist_entry, _head); diff --git a/plugins/in_node_exporter_metrics/ne_stat_linux.c b/plugins/in_node_exporter_metrics/ne_stat_linux.c index 46a5d89c1b2..64cc0e6eee0 100644 --- a/plugins/in_node_exporter_metrics/ne_stat_linux.c +++ b/plugins/in_node_exporter_metrics/ne_stat_linux.c @@ -83,7 +83,7 @@ static int stat_update(struct flb_ne *ctx) struct flb_slist_entry *s_val; mk_list_init(&list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/stat", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/stat", &list); if (ret == -1) { flb_plg_error(ctx->ins, "failed to read %s/stat", ctx->path_procfs); return -1; diff --git a/plugins/in_node_exporter_metrics/ne_thermalzone_linux.c b/plugins/in_node_exporter_metrics/ne_thermalzone_linux.c index 19a03f95517..a36e558707b 100644 --- a/plugins/in_node_exporter_metrics/ne_thermalzone_linux.c +++ b/plugins/in_node_exporter_metrics/ne_thermalzone_linux.c @@ -122,7 +122,7 @@ static int ne_thermalzone_update_thermal_zones(struct flb_ne *ctx) entry = mk_list_entry(head, struct flb_slist_entry, _head); /* Core ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "temp", NULL, &temp); @@ -130,7 +130,7 @@ static int ne_thermalzone_update_thermal_zones(struct flb_ne *ctx) continue; } - ret = ne_utils_file_read_sds(ctx->path_sysfs, entry->str, "type", NULL, &type); + ret = ne_utils_file_read_sds(ctx, ctx->path_sysfs, entry->str, "type", NULL, &type); if (ret != 0) { flb_plg_error(ctx->ins, "unable to get type for zone: %s", entry->str); continue; @@ -206,7 +206,7 @@ static int ne_thermalzone_update_cooling_devices(struct flb_ne *ctx) entry = mk_list_entry(head, struct flb_slist_entry, _head); /* Core ID */ - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "cur_state", NULL, &cur_state); @@ -214,7 +214,7 @@ static int ne_thermalzone_update_cooling_devices(struct flb_ne *ctx) continue; } - ret = ne_utils_file_read_uint64(ctx->path_sysfs, + ret = ne_utils_file_read_uint64(ctx, ctx->path_sysfs, entry->str, "max_state", NULL, &max_state); @@ -222,7 +222,7 @@ static int ne_thermalzone_update_cooling_devices(struct flb_ne *ctx) continue; } - ret = ne_utils_file_read_sds(ctx->path_sysfs, entry->str, "type", NULL, &type); + ret = ne_utils_file_read_sds(ctx, ctx->path_sysfs, entry->str, "type", NULL, &type); if (ret != 0) { flb_plg_error(ctx->ins, "unable to get type for zone: %s", entry->str); continue; diff --git a/plugins/in_node_exporter_metrics/ne_utils.c b/plugins/in_node_exporter_metrics/ne_utils.c index 8d3e8d6def5..591c856b54f 100644 --- a/plugins/in_node_exporter_metrics/ne_utils.c +++ b/plugins/in_node_exporter_metrics/ne_utils.c @@ -65,7 +65,8 @@ int ne_utils_str_to_uint64(char *str, uint64_t *out_val) return 0; } -int ne_utils_file_read_uint64(const char *mount, +int ne_utils_file_read_uint64(struct flb_ne *ctx, + const char *mount, const char *path, const char *join_a, const char *join_b, uint64_t *out_val) @@ -122,18 +123,30 @@ int ne_utils_file_read_uint64(const char *mount, fd = open(p, O_RDONLY); if (fd == -1) { + if (ctx) { + flb_plg_error(ctx->ins, "could not open '%s'", p); + } + else { + flb_errno(); + } flb_sds_destroy(p); return -1; } - flb_sds_destroy(p); bytes = read(fd, &tmp, sizeof(tmp)); if (bytes == -1) { - flb_errno(); + if (ctx) { + flb_plg_error(ctx->ins, "could not read from '%s'", p); + } + else { + flb_errno(); + } close(fd); + flb_sds_destroy(p); return -1; } close(fd); + flb_sds_destroy(p); ret = ne_utils_str_to_uint64(tmp, &val); if (ret == -1) { @@ -148,7 +161,7 @@ int ne_utils_file_read_uint64(const char *mount, * Read a file and every non-empty line is stored as a flb_slist_entry in the * given list. */ -int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list *list) +int ne_utils_file_read_lines(struct flb_ne *ctx, const char *mount, const char *path, struct mk_list *list) { int len; int ret; @@ -167,7 +180,12 @@ int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list snprintf(real_path, sizeof(real_path) - 1, "%s%s", mount, path); f = fopen(real_path, "r"); if (f == NULL) { - flb_errno(); + if (ctx) { + flb_plg_error(ctx->ins, "could not open '%s'", real_path); + } + else { + flb_errno(); + } return -1; } @@ -196,7 +214,8 @@ int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list /* * Read a file and store the first line as a string. */ -int ne_utils_file_read_sds(const char *mount, +int ne_utils_file_read_sds(struct flb_ne *ctx, + const char *mount, const char *path, const char *join_a, const char *join_b, @@ -250,18 +269,30 @@ int ne_utils_file_read_sds(const char *mount, fd = open(p, O_RDONLY); if (fd == -1) { + if (ctx) { + flb_plg_error(ctx->ins, "could not open '%s'", p); + } + else { + flb_errno(); + } flb_sds_destroy(p); return -1; } - flb_sds_destroy(p); bytes = read(fd, &tmp, sizeof(tmp)); if (bytes == -1) { - flb_errno(); + if (ctx) { + flb_plg_error(ctx->ins, "could not read from '%s'", p); + } + else { + flb_errno(); + } close(fd); + flb_sds_destroy(p); return -1; } close(fd); + flb_sds_destroy(p); for (i = bytes-1; i > 0; i--) { if (tmp[i] != '\n' && tmp[i] != '\r') { diff --git a/plugins/in_node_exporter_metrics/ne_utils.h b/plugins/in_node_exporter_metrics/ne_utils.h index aabd3c74dd3..a365fbd4937 100644 --- a/plugins/in_node_exporter_metrics/ne_utils.h +++ b/plugins/in_node_exporter_metrics/ne_utils.h @@ -28,18 +28,23 @@ int ne_utils_str_to_double(char *str, double *out_val); int ne_utils_str_to_uint64(char *str, uint64_t *out_val); -int ne_utils_file_read_uint64(const char *mount, +int ne_utils_file_read_uint64(struct flb_ne *ctx, + const char *mount, const char *path, const char *join_a, const char *join_b, uint64_t *out_val); -int ne_utils_file_read_sds(const char *mount, +int ne_utils_file_read_sds(struct flb_ne *ctx, + const char *mount, const char *path, const char *join_a, const char *join_b, flb_sds_t *str); -int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list *list); +int ne_utils_file_read_lines(struct flb_ne *ctx, + const char *mount, + const char *path, + struct mk_list *list); int ne_utils_path_scan(struct flb_ne *ctx, const char *mount, const char *path, int expected, struct mk_list *list); #endif diff --git a/plugins/in_node_exporter_metrics/ne_vmstat_linux.c b/plugins/in_node_exporter_metrics/ne_vmstat_linux.c index 183495bff22..5571c9488f8 100644 --- a/plugins/in_node_exporter_metrics/ne_vmstat_linux.c +++ b/plugins/in_node_exporter_metrics/ne_vmstat_linux.c @@ -64,7 +64,7 @@ static int vmstat_configure(struct flb_ne *ctx) mk_list_init(&list); mk_list_init(&split_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/vmstat", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/vmstat", &list); if (ret == -1) { return -1; } @@ -139,7 +139,7 @@ static int vmstat_update(struct flb_ne *ctx) mk_list_init(&list); mk_list_init(&split_list); - ret = ne_utils_file_read_lines(ctx->path_procfs, "/vmstat", &list); + ret = ne_utils_file_read_lines(ctx, ctx->path_procfs, "/vmstat", &list); if (ret == -1) { return -1; } diff --git a/plugins/in_splunk/splunk.c b/plugins/in_splunk/splunk.c index e0994d1518e..d7c339efdd6 100644 --- a/plugins/in_splunk/splunk.c +++ b/plugins/in_splunk/splunk.c @@ -261,6 +261,16 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_splunk, tag_key), "" }, + { + FLB_CONFIG_MAP_BOOL, "add_remote_addr", "false", + 0, FLB_TRUE, offsetof(struct flb_splunk, add_remote_addr), + "Inject a remote address using the X-Forwarded-For header or connection address" + }, + { + FLB_CONFIG_MAP_STR, "remote_addr_key", "remote_addr", + 0, FLB_TRUE, offsetof(struct flb_splunk, remote_addr_key), + "Set a record key for storing the remote address" + }, /* EOF */ diff --git a/plugins/in_splunk/splunk.h b/plugins/in_splunk/splunk.h index 56a45bf2d05..93db1f444bf 100644 --- a/plugins/in_splunk/splunk.h +++ b/plugins/in_splunk/splunk.h @@ -54,6 +54,8 @@ struct flb_splunk { size_t ingested_auth_header_len; int store_token_in_metadata; flb_sds_t store_token_key; + int add_remote_addr; + flb_sds_t remote_addr_key; struct flb_log_event_encoder log_encoder; diff --git a/plugins/in_splunk/splunk_prot.c b/plugins/in_splunk/splunk_prot.c index 536a939f108..0d81ef26304 100644 --- a/plugins/in_splunk/splunk_prot.c +++ b/plugins/in_splunk/splunk_prot.c @@ -20,10 +20,14 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include #include #include @@ -155,6 +159,152 @@ static int send_json_message_response(struct splunk_conn *conn, int http_status, return 0; } +/* + * We use two backends for HTTP parsing and it depends on the version of the + * protocol: + * + * http/1.x: we use Monkey HTTP parser: struct mk_http_session.parser + */ +static int http_header_lookup(int version, void *ptr, char *key, + char **val, size_t *val_len) +{ + int key_len; + + /* HTTP/1.1 */ + struct mk_list *head; + struct mk_http_session *session; + struct mk_http_request *request_11; + struct mk_http_header *header; + + /* HTTP/2.0 */ + char *value; + struct flb_http_request *request_20; + + if (!key) { + return -1; + } + + key_len = strlen(key); + if (key_len <= 0) { + return -1; + } + + if (version <= HTTP_PROTOCOL_VERSION_11) { + if (!ptr) { + return -1; + } + + request_11 = (struct mk_http_request *) ptr; + session = request_11->session; + mk_list_foreach(head, &session->parser.header_list) { + header = mk_list_entry(head, struct mk_http_header, _head); + if (header->key.len == key_len && + strncasecmp(header->key.data, key, key_len) == 0) { + *val = header->val.data; + *val_len = header->val.len; + return 0; + } + } + return -1; + } + else if (version == HTTP_PROTOCOL_VERSION_20) { + request_20 = ptr; + if (!request_20) { + return -1; + } + + value = flb_http_request_get_header(request_20, key); + if (!value) { + return -1; + } + + *val = value; + *val_len = strlen(value); + return 0; + } + + return -1; +} + +static void extract_xff_value(const char *value, size_t value_len, + const char **out_value, size_t *out_len) +{ + const char *start; + const char *end; + const char *comma; + + *out_value = NULL; + *out_len = 0; + + if (value == NULL || value_len == 0) { + return; + } + + start = value; + end = value + value_len; + + while (start < end && (*start == ' ' || *start == '\t')) { + start++; + } + + comma = memchr(start, ',', end - start); + if (comma != NULL) { + end = comma; + } + + while (end > start && (end[-1] == ' ' || end[-1] == '\t')) { + end--; + } + + if (end > start) { + *out_value = start; + *out_len = end - start; + } +} + +static int extract_remote_address(const char *xff_value, + size_t xff_value_len, + struct flb_connection *connection, + char **out, + size_t *out_len) +{ + const char *value = NULL; + size_t len = 0; + + extract_xff_value(xff_value, xff_value_len, &value, &len); + + if (value == NULL && connection != NULL) { + value = flb_connection_get_remote_address(connection); + if (value != NULL) { + len = strlen(value); + } + } + + if (value == NULL || len == 0) { + return -1; + } + + *out = value; + *out_len = len; + return 0; +} + +static int append_remote_addr(struct flb_splunk *ctx, + const char *addr, + size_t addr_len) +{ + if (ctx->add_remote_addr != FLB_TRUE || + ctx->remote_addr_key == NULL || + addr == NULL || addr_len == 0) { + return FLB_EVENT_ENCODER_SUCCESS; + } + + return flb_log_event_encoder_append_body_values( + &ctx->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE(ctx->remote_addr_key), + FLB_LOG_EVENT_STRING_VALUE(addr, addr_len)); +} + /* implements functionality to get tag from key in record */ static flb_sds_t tag_key(struct flb_splunk *ctx, msgpack_object *map) { @@ -191,7 +341,10 @@ static flb_sds_t tag_key(struct flb_splunk *ctx, msgpack_object *map) * Process a raw text payload for Splunk HEC requests, uses the delimited character to split records, * return the number of processed bytes */ -static int process_raw_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char *buf, size_t size) +static int process_raw_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, + char *buf, size_t size, + const char *remote_addr, + size_t remote_addr_len) { int ret = FLB_EVENT_ENCODER_SUCCESS; @@ -236,6 +389,12 @@ static int process_raw_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char } } + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = append_remote_addr(ctx, + remote_addr, + remote_addr_len); + } + if (ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_commit_record(&ctx->log_encoder); } @@ -267,7 +426,10 @@ static int process_raw_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *record, flb_sds_t tag, flb_sds_t tag_from_record, - struct flb_time tm) { + struct flb_time tm, + const char *remote_addr, + size_t remote_addr_len) +{ int ret; int i; msgpack_object_kv *kv; @@ -280,13 +442,25 @@ static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *recor &tm); } - if (ctx->store_token_in_metadata == FLB_TRUE) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - record); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + /* Always build body by appending map entries so we can extend it */ + if (record->type == MSGPACK_OBJECT_MAP) { + kv = record->via.map.ptr; + for (i = 0; i < record->via.map.size && + ret == FLB_EVENT_ENCODER_SUCCESS; i++) { + ret = flb_log_event_encoder_append_body_values( + &ctx->log_encoder, + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); + } + } + else { + ret = flb_log_event_encoder_set_body_from_msgpack_object(&ctx->log_encoder, + record); } + } + if (ctx->store_token_in_metadata == FLB_TRUE) { if (ctx->ingested_auth_header != NULL) { if (ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_metadata_values( @@ -299,15 +473,6 @@ static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *recor } else { if (ctx->ingested_auth_header != NULL) { - /* iterate through the old record map to create the appendable new buffer */ - kv = record->via.map.ptr; - for(i = 0; i < record->via.map.size && ret == FLB_EVENT_ENCODER_SUCCESS; i++) { - ret = flb_log_event_encoder_append_body_values( - &ctx->log_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&kv[i].val)); - } - if (ret == FLB_EVENT_ENCODER_SUCCESS) { ret = flb_log_event_encoder_append_body_values( &ctx->log_encoder, @@ -316,13 +481,10 @@ static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *recor ctx->ingested_auth_header_len)); } } - else { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - &ctx->log_encoder, - record); - } - } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = append_remote_addr(ctx, remote_addr, remote_addr_len); } if (ret == FLB_EVENT_ENCODER_SUCCESS) { @@ -358,7 +520,10 @@ static void process_flb_log_append(struct flb_splunk *ctx, msgpack_object *recor } } -static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char *buf, size_t size) +static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, + char *buf, size_t size, + const char *remote_addr, + size_t remote_addr_len) { size_t off = 0; msgpack_unpacked result; @@ -378,7 +543,9 @@ static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char tag_from_record = tag_key(ctx, &result.data); } - process_flb_log_append(ctx, &result.data, tag, tag_from_record, tm); + process_flb_log_append(ctx, &result.data, tag, tag_from_record, tm, + remote_addr, + remote_addr_len); flb_log_event_encoder_reset(&ctx->log_encoder); } @@ -393,7 +560,9 @@ static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char tag_from_record = tag_key(ctx, &record); } - process_flb_log_append(ctx, &record, tag, tag_from_record, tm); + process_flb_log_append(ctx, &record, tag, tag_from_record, tm, + remote_addr, + remote_addr_len); /* TODO : Optimize this * @@ -423,7 +592,9 @@ static int process_json_payload_pack(struct flb_splunk *ctx, flb_sds_t tag, char } static ssize_t parse_hec_payload_json(struct flb_splunk *ctx, flb_sds_t tag, - char *payload, size_t size) + char *payload, size_t size, + const char *remote_addr, + size_t remote_addr_len) { int ret; int out_size; @@ -452,7 +623,8 @@ static ssize_t parse_hec_payload_json(struct flb_splunk *ctx, flb_sds_t tag, } /* Process the packaged JSON and return the last byte used */ - process_json_payload_pack(ctx, tag, pack, out_size); + process_json_payload_pack(ctx, tag, pack, out_size, + remote_addr, remote_addr_len); flb_free(pack); return 0; @@ -510,22 +682,28 @@ static int validate_auth_header(struct flb_splunk *ctx, struct mk_http_request * } static int handle_hec_payload(struct flb_splunk *ctx, int content_type, - flb_sds_t tag, char *buf, size_t size) + flb_sds_t tag, char *buf, size_t size, + const char *remote_addr, + size_t remote_addr_len) { int ret = -1; if (content_type == HTTP_CONTENT_JSON) { - ret = parse_hec_payload_json(ctx, tag, buf, size); + ret = parse_hec_payload_json(ctx, tag, buf, size, + remote_addr, remote_addr_len); } else if (content_type == HTTP_CONTENT_TEXT) { - ret = process_raw_payload_pack(ctx, tag, buf, size); + ret = process_raw_payload_pack(ctx, tag, buf, size, + remote_addr, remote_addr_len); } else if (content_type == HTTP_CONTENT_UNKNOWN) { if (buf[0] == '{') { - ret = parse_hec_payload_json(ctx, tag, buf, size); + ret = parse_hec_payload_json(ctx, tag, buf, size, + remote_addr, remote_addr_len); } else { - ret = process_raw_payload_pack(ctx, tag, buf, size); + ret = process_raw_payload_pack(ctx, tag, buf, size, + remote_addr, remote_addr_len); } } @@ -535,7 +713,9 @@ static int handle_hec_payload(struct flb_splunk *ctx, int content_type, static int process_hec_payload(struct flb_splunk *ctx, struct splunk_conn *conn, flb_sds_t tag, struct mk_http_session *session, - struct mk_http_request *request) + struct mk_http_request *request, + const char *remote_addr, + size_t remote_addr_len) { int i = 0; int ret = 0; @@ -603,11 +783,13 @@ static int process_hec_payload(struct flb_splunk *ctx, struct splunk_conn *conn, return -1; } - ret = handle_hec_payload(ctx, type, tag, gz_data, gz_size); + ret = handle_hec_payload(ctx, type, tag, gz_data, gz_size, + remote_addr, remote_addr_len); flb_free(gz_data); } else { - ret = handle_hec_payload(ctx, type, tag, request->data.data, request->data.len); + ret = handle_hec_payload(ctx, type, tag, request->data.data, request->data.len, + remote_addr, remote_addr_len); } return ret; @@ -616,7 +798,9 @@ static int process_hec_payload(struct flb_splunk *ctx, struct splunk_conn *conn, static int process_hec_raw_payload(struct flb_splunk *ctx, struct splunk_conn *conn, flb_sds_t tag, struct mk_http_session *session, - struct mk_http_request *request) + struct mk_http_request *request, + const char *remote_addr, + size_t remote_addr_len) { int ret = -1; struct mk_http_header *header; @@ -647,7 +831,8 @@ static int process_hec_raw_payload(struct flb_splunk *ctx, struct splunk_conn *c } /* Always handle as raw type of payloads here */ - ret = process_raw_payload_pack(ctx, tag, request->data.data, request->data.len); + ret = process_raw_payload_pack(ctx, tag, request->data.data, request->data.len, + remote_addr, remote_addr_len); return ret; } @@ -691,6 +876,11 @@ int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, off_t diff; flb_sds_t tag; struct mk_http_header *header; + char *hval = NULL; + size_t hlen = 0; + const char *peer; + const char *remote_addr = NULL; + size_t remote_addr_len = 0; if (request->uri.data[0] != '/') { send_response(conn, 400, "error: invalid request\n"); @@ -835,13 +1025,28 @@ int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, request->data.len = out_chunked_size; } + if (http_header_lookup(HTTP_PROTOCOL_VERSION_11, request, + SPLUNK_XFF_HEADER, &hval, &hlen) == 0) { + extract_remote_address(hval, hlen, conn->connection, + (char **) &remote_addr, + &remote_addr_len); + } + if (remote_addr == NULL || remote_addr_len == 0) { + peer = flb_connection_get_remote_address(conn->connection); + if (peer != NULL) { + remote_addr = peer; + remote_addr_len = strlen(peer); + } + } + /* Handle every ingested payload cleanly */ flb_log_event_encoder_reset(&ctx->log_encoder); if (request->method == MK_METHOD_POST) { if (strcasecmp(uri, "/services/collector/raw/1.0") == 0 || strcasecmp(uri, "/services/collector/raw") == 0) { - ret = process_hec_raw_payload(ctx, conn, tag, session, request); + ret = process_hec_raw_payload(ctx, conn, tag, session, request, + remote_addr, remote_addr_len); if (ret == -2) { /* Response already sent, skip further response */ @@ -866,7 +1071,8 @@ int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, strcasecmp(uri, "/services/collector/event") == 0 || strcasecmp(uri, "/services/collector") == 0) { - ret = process_hec_payload(ctx, conn, tag, session, request); + ret = process_hec_payload(ctx, conn, tag, session, request, + remote_addr, remote_addr_len); if (ret == -2) { flb_sds_destroy(tag); mk_mem_free(uri); @@ -1056,7 +1262,9 @@ static int validate_auth_header_ng(struct flb_splunk *ctx, struct flb_http_reque static int process_hec_payload_ng(struct flb_http_request *request, struct flb_http_response *response, flb_sds_t tag, - struct flb_splunk *ctx) + struct flb_splunk *ctx, + const char *remote_addr, + size_t remote_addr_len) { int type = -1; int ret = 0; @@ -1092,13 +1300,16 @@ static int process_hec_payload_ng(struct flb_http_request *request, return -2; } - return handle_hec_payload(ctx, type, tag, request->body, cfl_sds_len(request->body)); + return handle_hec_payload(ctx, type, tag, request->body, cfl_sds_len(request->body), + remote_addr, remote_addr_len); } static int process_hec_raw_payload_ng(struct flb_http_request *request, struct flb_http_response *response, flb_sds_t tag, - struct flb_splunk *ctx) + struct flb_splunk *ctx, + const char *remote_addr, + size_t remote_addr_len) { int ret = 0; size_t size = 0; @@ -1129,7 +1340,8 @@ static int process_hec_raw_payload_ng(struct flb_http_request *request, } /* Always handle as raw type of payloads here */ - return process_raw_payload_pack(ctx, tag, request->body, cfl_sds_len(request->body)); + return process_raw_payload_pack(ctx, tag, request->body, cfl_sds_len(request->body), + remote_addr, remote_addr_len); } int splunk_prot_handle_ng(struct flb_http_request *request, @@ -1138,6 +1350,12 @@ int splunk_prot_handle_ng(struct flb_http_request *request, struct flb_splunk *context; int ret = -1; flb_sds_t tag; + struct flb_http_server_session *parent_session; + char *hval = NULL; + size_t hlen = 0; + const char *peer; + const char *remote_addr = NULL; + size_t remote_addr_len = 0; context = (struct flb_splunk *) response->stream->user_data; @@ -1185,6 +1403,23 @@ int splunk_prot_handle_ng(struct flb_http_request *request, /* Handle every ingested payload cleanly */ flb_log_event_encoder_reset(&context->log_encoder); + parent_session = (struct flb_http_server_session *) request->stream->parent; + if (parent_session != NULL) { + if (http_header_lookup(HTTP_PROTOCOL_VERSION_20, request, + SPLUNK_XFF_HEADER, &hval, &hlen) == 0) { + extract_remote_address(hval, hlen, parent_session->connection, + (char **) &remote_addr, + &remote_addr_len); + } + if (remote_addr == NULL || remote_addr_len == 0) { + peer = flb_connection_get_remote_address(parent_session->connection); + if (peer != NULL) { + remote_addr = peer; + remote_addr_len = strlen(peer); + } + } + } + if (request->method != HTTP_METHOD_POST) { /* HEAD, PUT, PATCH, and DELETE methods are prohibited to use.*/ send_response_ng(response, 400, "error: invalid HTTP method\n"); @@ -1200,7 +1435,8 @@ int splunk_prot_handle_ng(struct flb_http_request *request, if (strcasecmp(request->path, "/services/collector/raw/1.0") == 0 || strcasecmp(request->path, "/services/collector/raw") == 0) { - ret = process_hec_raw_payload_ng(request, response, tag, context); + ret = process_hec_raw_payload_ng(request, response, tag, context, + remote_addr, remote_addr_len); if (ret == -2) { /* Response already sent, skip further response */ flb_sds_destroy(tag); @@ -1218,7 +1454,8 @@ int splunk_prot_handle_ng(struct flb_http_request *request, else if (strcasecmp(request->path, "/services/collector/event/1.0") == 0 || strcasecmp(request->path, "/services/collector/event") == 0 || strcasecmp(request->path, "/services/collector") == 0) { - ret = process_hec_payload_ng(request, response, tag, context); + ret = process_hec_payload_ng(request, response, tag, context, + remote_addr, remote_addr_len); if (ret == -2) { /* Response already sent, skip further response */ flb_sds_destroy(tag); @@ -1239,5 +1476,6 @@ int splunk_prot_handle_ng(struct flb_http_request *request, } flb_sds_destroy(tag); + return ret; } diff --git a/plugins/in_splunk/splunk_prot.h b/plugins/in_splunk/splunk_prot.h index f979d4755a8..5c965155686 100644 --- a/plugins/in_splunk/splunk_prot.h +++ b/plugins/in_splunk/splunk_prot.h @@ -25,6 +25,8 @@ #define SPLUNK_AUTH_MISSING_CRED -1 #define SPLUNK_AUTH_UNAUTHORIZED -2 +#define SPLUNK_XFF_HEADER "x-forwarded-for" + #include int splunk_prot_handle(struct flb_splunk *ctx, struct splunk_conn *conn, diff --git a/plugins/in_syslog/syslog_prot.c b/plugins/in_syslog/syslog_prot.c index d1b4c0c87d7..f2624e3727f 100644 --- a/plugins/in_syslog/syslog_prot.c +++ b/plugins/in_syslog/syslog_prot.c @@ -240,7 +240,7 @@ int syslog_prot_process(struct syslog_conn *conn) char *sp = p; size_t n = 0; while (sp < end && *sp >= '0' && *sp <= '9') { - if (n > SIZE_MAX / 10) { + if (n >= SIZE_MAX / 10) { n = SIZE_MAX; break; } diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c index a7637003c05..ca032f13680 100644 --- a/plugins/in_tail/tail_config.c +++ b/plugins/in_tail/tail_config.c @@ -501,6 +501,13 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, "Total number of truncated occurences for long lines", 1, (char *[]) {"name"}); + ctx->cmt_long_line_skipped = + cmt_counter_create(ins->cmt, + "fluentbit", "input", + "long_line_skipped_total", + "Total number of skipped occurences for long lines", + 1, (char *[]) {"name"}); + /* OLD metrics */ flb_metrics_add(FLB_TAIL_METRIC_F_OPENED, "files_opened", ctx->ins->metrics); @@ -512,6 +519,8 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, "multiline_truncated", ctx->ins->metrics); flb_metrics_add(FLB_TAIL_METRIC_L_TRUNCATED, "long_line_truncated", ctx->ins->metrics); + flb_metrics_add(FLB_TAIL_METRIC_L_SKIPPED, + "long_line_skipped", ctx->ins->metrics); #endif return ctx; diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h index ce5afad52b5..3f54af93e42 100644 --- a/plugins/in_tail/tail_config.h +++ b/plugins/in_tail/tail_config.h @@ -43,6 +43,7 @@ #define FLB_TAIL_METRIC_F_ROTATED 102 /* number of rotated files */ #define FLB_TAIL_METRIC_M_TRUNCATED 103 /* number of truncated occurrences of multiline */ #define FLB_TAIL_METRIC_L_TRUNCATED 104 /* number of truncated occurrences of long lines */ +#define FLB_TAIL_METRIC_L_SKIPPED 105 /* number of skipped occurrences of long lines */ #endif struct flb_tail_config { @@ -166,12 +167,15 @@ struct flb_tail_config { struct flb_log_event_encoder log_event_encoder; struct flb_log_event_decoder log_event_decoder; +#ifdef FLB_HAVE_METRICS /* Metrics */ struct cmt_counter *cmt_files_opened; struct cmt_counter *cmt_files_closed; struct cmt_counter *cmt_files_rotated; struct cmt_counter *cmt_multiline_truncated; struct cmt_counter *cmt_long_line_truncated; + struct cmt_counter *cmt_long_line_skipped; +#endif /* Hash: hash tables for quick acess to registered files */ struct flb_hash_table *static_hash; diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c index aac73a68764..e553378ada8 100644 --- a/plugins/in_tail/tail_file.c +++ b/plugins/in_tail/tail_file.c @@ -1131,7 +1131,17 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, ssize_t offset, struct flb_tail_config *ctx) { - int fd; + /* + * Cleanup uses staged-goto labels in reverse acquisition order; + * jumping to err_X cleans up everything acquired up to and including X + * and falls through to subsequent labels. Each label undoes exactly one + * acquisition step. Construction-failure paths must NOT route through + * flb_tail_file_remove() because that helper increments the + * cmt_files_closed counter unconditionally and would unbalance the + * cmt_files_opened/closed pair (the matching opened increment lives at + * the very end of this function, immediately before return 0). + */ + int fd = -1; int ret; uint64_t stream_id; uint64_t ts; @@ -1169,12 +1179,14 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file = flb_calloc(1, sizeof(struct flb_tail_file)); if (!file) { flb_errno(); - goto error; + goto err_close_fd; } /* Initialize */ - file->watch_fd = -1; + file->config = ctx; file->fd = fd; + file->watch_fd = -1; + file->tail_mode = mode; /* On non-windows environments check if the original path is a link */ ret = lstat(path, &lst); @@ -1189,7 +1201,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, ret = stat_to_hash_bits(ctx, st, &hash_bits); if (ret != 0) { flb_plg_error(ctx->ins, "error procesisng hash bits for file %s", path); - goto error; + goto err_free_file; } file->hash_bits = hash_bits; @@ -1197,7 +1209,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, ret = stat_to_hash_key(ctx, st, &hash_key); if (ret != 0) { flb_plg_error(ctx->ins, "error procesisng hash key for file %s", path); - goto error; + goto err_free_file; } file->hash_key = hash_key; @@ -1206,8 +1218,6 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->size = st->st_size; file->buf_len = 0; file->parsed = 0; - file->config = ctx; - file->tail_mode = mode; file->tag_len = 0; file->tag_buf = NULL; file->rotated = 0; @@ -1229,7 +1239,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, ctx->buf_max_size); if (file->decompression_context == NULL) { - goto error; + goto err_free_hash_key; } } @@ -1247,7 +1257,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, ret = flb_tail_file_name_dup(path, file); if (!file->name) { flb_errno(); - goto error; + goto err_free_decomp; } /* We keep a copy of the initial filename in orig_name. This is required @@ -1255,9 +1265,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->orig_name = flb_strdup(file->name); if (!file->orig_name) { flb_errno(); - flb_free(file->name); - file->name = NULL; - goto error; + goto err_free_name; } file->orig_name_len = file->name_len; @@ -1271,7 +1279,15 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->dmode_flush_timeout = 0; file->dmode_complete = true; file->dmode_buf = flb_sds_create_size(ctx->docker_mode == FLB_TRUE ? 65536 : 0); + if (!file->dmode_buf) { + flb_errno(); + goto err_destroy_sbuf; + } file->dmode_lastline = flb_sds_create_size(ctx->docker_mode == FLB_TRUE ? 20000 : 0); + if (!file->dmode_lastline) { + flb_errno(); + goto err_free_dmode_buf; + } file->dmode_firstline = false; #ifdef FLB_HAVE_SQLDB file->db_id = 0; @@ -1302,7 +1318,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, "could not create multiline stream for file: %s", inode_str); flb_sds_destroy(inode_str); - goto error; + goto err_free_dmode_lastline; } file->ml_stream_id = stream_id; flb_sds_destroy(inode_str); @@ -1323,7 +1339,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->buf_data = flb_malloc(file->buf_size); if (!file->buf_data) { flb_errno(); - goto error; + goto err_destroy_ml_stream; } /* Initialize (optional) dynamic tag */ @@ -1333,7 +1349,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, if (!tag) { flb_errno(); flb_plg_error(ctx->ins, "failed to allocate tag buffer"); - goto error; + goto err_free_buf_data; } #ifdef FLB_HAVE_REGEX ret = tag_compose(ctx->ins->tag, ctx->tag_regex, path, tag, &tag_len, ctx); @@ -1347,7 +1363,7 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, flb_free(tag); if (ret != 0) { flb_plg_error(ctx->ins, "failed to compose tag for file: %s", path); - goto error; + goto err_free_buf_data; } } else { @@ -1357,83 +1373,138 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, if (!file->tag_buf) { flb_plg_error(ctx->ins, "failed to set tag for file: %s", path); flb_errno(); - goto error; + goto err_free_buf_data; } if (mode == FLB_TAIL_STATIC) { mk_list_add(&file->_head, &ctx->files_static); ctx->files_static_count++; - flb_hash_table_add(ctx->static_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); + + ret = flb_hash_table_add(ctx->static_hash, + file->hash_key, flb_sds_len(file->hash_key), + file, sizeof(file)); + if (ret < 0) { + flb_plg_error(ctx->ins, "could not register file in static hash"); + goto err_unlist; + } + tail_signal_manager(file->config); } else if (mode == FLB_TAIL_EVENT) { mk_list_add(&file->_head, &ctx->files_event); - flb_hash_table_add(ctx->event_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); + + ret = flb_hash_table_add(ctx->event_hash, + file->hash_key, flb_sds_len(file->hash_key), + file, sizeof(file)); + if (ret < 0) { + flb_plg_error(ctx->ins, "could not register file in event hash"); + goto err_unlist; + } /* Register this file into the fs_event monitoring */ ret = flb_tail_fs_add(ctx, file); if (ret == -1) { flb_plg_error(ctx->ins, "could not register file into fs_events"); - goto error; + goto err_unlist; } } /* Set the file position (database offset, head or tail) */ ret = set_file_position(ctx, file); if (ret == -1) { - flb_tail_file_remove(file); - goto error; + goto err_fs_remove; } /* Remaining bytes to read */ file->pending_bytes = file->size - file->offset; -#ifdef FLB_HAVE_METRICS - name = (char *) flb_input_name(ctx->ins); - ts = cfl_time_now(); - cmt_counter_inc(ctx->cmt_files_opened, ts, 1, (char *[]) {name}); - - /* Old api */ - flb_metrics_sum(FLB_TAIL_METRIC_F_OPENED, 1, ctx->ins->metrics); -#endif - file->sl_log_event_encoder = flb_log_event_encoder_create( FLB_LOG_EVENT_FORMAT_DEFAULT); if (file->sl_log_event_encoder == NULL) { - flb_tail_file_remove(file); - - goto error; + goto err_set_pos; } file->ml_log_event_encoder = flb_log_event_encoder_create( FLB_LOG_EVENT_FORMAT_DEFAULT); if (file->ml_log_event_encoder == NULL) { - flb_tail_file_remove(file); - - goto error; + goto err_destroy_sl; } flb_plg_debug(ctx->ins, "inode=%"PRIu64" with offset=%"PRId64" appended as %s", file->inode, file->offset, path); + +#ifdef FLB_HAVE_METRICS + name = (char *) flb_input_name(ctx->ins); + ts = cfl_time_now(); + cmt_counter_inc(ctx->cmt_files_opened, ts, 1, (char *[]) {name}); + + /* Old api */ + flb_metrics_sum(FLB_TAIL_METRIC_F_OPENED, 1, ctx->ins->metrics); +#endif + return 0; -error: - if (file) { - if (file->buf_data) { - flb_free(file->buf_data); - } - if (file->name) { - flb_free(file->name); +/* + * Cleanup ladder: each label undoes exactly one acquisition and falls + * through to the next. Failed-acquisition gotos jump to the label that + * cleans up the LAST resource that was successfully acquired, skipping + * the cleanup of the resource that was never created. + */ +err_destroy_sl: + flb_log_event_encoder_destroy(file->sl_log_event_encoder); +err_set_pos: + /* set_file_position has no resource of its own; fall through */ +err_fs_remove: + if (mode == FLB_TAIL_EVENT) { + flb_tail_fs_remove(ctx, file); + } +err_unlist: + if (mode == FLB_TAIL_STATIC) { + mk_list_del(&file->_head); + if (ctx->files_static_count > 0) { + ctx->files_static_count--; } - flb_free(file); + flb_hash_table_del(ctx->static_hash, file->hash_key); + } + else if (mode == FLB_TAIL_EVENT) { + mk_list_del(&file->_head); + flb_hash_table_del(ctx->event_hash, file->hash_key); + } + flb_free(file->tag_buf); +err_free_buf_data: + flb_free(file->buf_data); +err_destroy_ml_stream: + if (ctx->ml_ctx && file->ml_stream_id > 0) { + flb_ml_stream_id_destroy_all(ctx->ml_ctx, file->ml_stream_id); + } +err_free_dmode_lastline: + flb_sds_destroy(file->dmode_lastline); +err_free_dmode_buf: + flb_sds_destroy(file->dmode_buf); +err_destroy_sbuf: + msgpack_sbuffer_destroy(&file->mult_sbuf); + /* orig_name was acquired immediately after name with no fallible step + * between, so its cleanup is folded into err_destroy_sbuf rather than + * carrying a label that no goto can reach */ + flb_free(file->orig_name); +err_free_name: + flb_free(file->name); + if (file->real_name) { + flb_free(file->real_name); } +err_free_decomp: + if (file->decompression_context) { + flb_decompression_context_destroy(file->decompression_context); + } +err_free_hash_key: + flb_sds_destroy(file->hash_key); +err_free_file: + flb_free(file); +err_close_fd: close(fd); - return -1; } @@ -1482,6 +1553,11 @@ void flb_tail_file_remove(struct flb_tail_file *file) flb_sds_destroy(file->dmode_buf); flb_sds_destroy(file->dmode_lastline); + + if (file->tail_mode == FLB_TAIL_STATIC && ctx->files_static_count > 0) { + ctx->files_static_count--; + } + mk_list_del(&file->_head); flb_tail_fs_remove(ctx, file); @@ -1643,6 +1719,18 @@ int flb_tail_file_chunk(struct flb_tail_file *file) return FLB_TAIL_ERROR; } +#ifdef FLB_HAVE_METRICS + if (file->skip_next == FLB_FALSE) { + cmt_counter_inc(ctx->cmt_long_line_skipped, + cfl_time_now(), 1, + (char *[]) { (char *) flb_input_name(ctx->ins) }); + + /* Old API */ + flb_metrics_sum(FLB_TAIL_METRIC_L_SKIPPED, 1, ctx->ins->metrics); + + } +#endif + /* Warn the user */ if (file->skip_warn == FLB_FALSE) { flb_plg_warn(ctx->ins, "file=%s have long lines. " @@ -1944,14 +2032,21 @@ int flb_tail_file_to_event(struct flb_tail_file *file) return -1; } + ret = flb_hash_table_add(ctx->event_hash, + file->hash_key, flb_sds_len(file->hash_key), + file, sizeof(file)); + if (ret < 0) { + flb_plg_error(ctx->ins, "could not register file in event hash"); + flb_tail_fs_remove(ctx, file); + return -1; + } + /* List swap: change from 'static' to 'event' list */ mk_list_del(&file->_head); ctx->files_static_count--; flb_hash_table_del(ctx->static_hash, file->hash_key); mk_list_add(&file->_head, &file->config->files_event); - flb_hash_table_add(ctx->event_hash, file->hash_key, flb_sds_len(file->hash_key), - file, sizeof(file)); file->tail_mode = FLB_TAIL_EVENT; diff --git a/plugins/in_tail/tail_fs_stat.c b/plugins/in_tail/tail_fs_stat.c index 74b6b95dc77..18f3fb13695 100644 --- a/plugins/in_tail/tail_fs_stat.c +++ b/plugins/in_tail/tail_fs_stat.c @@ -246,8 +246,9 @@ int flb_tail_fs_stat_add(struct flb_tail_file *file) int flb_tail_fs_stat_remove(struct flb_tail_file *file) { - if (file->tail_mode == FLB_TAIL_EVENT) { + if (file->fs_backend != NULL) { flb_free(file->fs_backend); + file->fs_backend = NULL; } return 0; } diff --git a/plugins/in_windows_exporter_metrics/we.h b/plugins/in_windows_exporter_metrics/we.h index e991a82730c..bf8b9f1c2e0 100644 --- a/plugins/in_windows_exporter_metrics/we.h +++ b/plugins/in_windows_exporter_metrics/we.h @@ -301,6 +301,7 @@ struct flb_we { /* WMI locator and service contexts */ IWbemLocator *locator; IWbemServices *service; + IWbemContext *wmi_context; float windows_version; diff --git a/plugins/in_windows_exporter_metrics/we_util.c b/plugins/in_windows_exporter_metrics/we_util.c index 6258727092e..c8af61eafc6 100644 --- a/plugins/in_windows_exporter_metrics/we_util.c +++ b/plugins/in_windows_exporter_metrics/we_util.c @@ -152,7 +152,7 @@ wchar_t* we_convert_str(char *str) return NULL; } - buf = flb_calloc(1, sizeof(PWSTR) * size); + buf = flb_calloc(1, sizeof(wchar_t) * size); if (buf == NULL) { flb_errno(); return NULL; diff --git a/plugins/in_windows_exporter_metrics/we_wmi.c b/plugins/in_windows_exporter_metrics/we_wmi.c index 03505c4bc90..cf9377c2dd4 100644 --- a/plugins/in_windows_exporter_metrics/we_wmi.c +++ b/plugins/in_windows_exporter_metrics/we_wmi.c @@ -34,6 +34,10 @@ static int wmi_coinitialize(struct flb_we *ctx, char* wmi_namespace) IWbemServices *service = 0; HRESULT hr; wchar_t *wnamespace; + BSTR bstrNamespace; + SYSTEM_INFO sysInfo; + VARIANT vArch; + VARIANT vReq; flb_plg_debug(ctx->ins, "initializing WMI instance...."); @@ -69,27 +73,62 @@ static int wmi_coinitialize(struct flb_we *ctx, char* wmi_namespace) } ctx->locator = locator; + hr = CoCreateInstance(&CLSID_WbemContext, 0, CLSCTX_INPROC_SERVER, &IID_IWbemContext, (LPVOID *) &ctx->wmi_context); + if (SUCCEEDED(hr) && ctx->wmi_context != NULL) { + GetNativeSystemInfo(&sysInfo); + + VariantInit(&vArch); + vArch.vt = VT_I4; + + if (sysInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64 || + sysInfo.wProcessorArchitecture == PROCESSOR_ARCHITECTURE_ARM64) { + vArch.lVal = 64; + } + else { + vArch.lVal = 32; + } + + ctx->wmi_context->lpVtbl->SetValue(ctx->wmi_context, L"__ProviderArchitecture", 0, &vArch); + VariantClear(&vArch); + + VariantInit(&vReq); + vReq.vt = VT_BOOL; + vReq.boolVal = VARIANT_TRUE; + ctx->wmi_context->lpVtbl->SetValue(ctx->wmi_context, L"__RequiredArchitecture", 0, &vReq); + VariantClear(&vReq); + } + if (wmi_namespace == NULL) { wnamespace = we_convert_str("ROOT\\CIMV2"); } else { wnamespace = we_convert_str(wmi_namespace); } + + bstrNamespace = SysAllocString(wnamespace); + /* Connect WMI server */ hr = locator->lpVtbl->ConnectServer(locator, - wnamespace, + bstrNamespace, NULL, NULL, 0, 0, 0, - NULL, + ctx->wmi_context, &service); + + SysFreeString(bstrNamespace); flb_free(wnamespace); if (FAILED(hr)) { flb_plg_error(ctx->ins, "Could not connect. Error code = %x", hr); locator->lpVtbl->Release(locator); + ctx->locator = NULL; + if (ctx->wmi_context != NULL) { + ctx->wmi_context->lpVtbl->Release(ctx->wmi_context); + ctx->wmi_context = NULL; + } CoUninitialize(); return hr; } @@ -109,6 +148,12 @@ static int wmi_coinitialize(struct flb_we *ctx, char* wmi_namespace) flb_plg_error(ctx->ins, "Could not set proxy blanket. Error code = %x", hr); service->lpVtbl->Release(service); locator->lpVtbl->Release(locator); + ctx->service = NULL; + ctx->locator = NULL; + if (ctx->wmi_context != NULL) { + ctx->wmi_context->lpVtbl->Release(ctx->wmi_context); + ctx->wmi_context = NULL; + } CoUninitialize(); return hr; } @@ -299,6 +344,8 @@ static inline int wmi_execute_query(struct flb_we *ctx, struct wmi_query_spec *s char *query = NULL; IEnumWbemClassObject* enumerator = NULL; size_t size; + BSTR bstrLanguage; + BSTR bstrQuery; size = 14 + strlen(spec->wmi_counter); if (spec->where_clause != NULL) { @@ -319,14 +366,20 @@ static inline int wmi_execute_query(struct flb_we *ctx, struct wmi_query_spec *s wquery = we_convert_str(query); flb_free(query); + bstrLanguage = SysAllocString(L"WQL"); + bstrQuery = SysAllocString(wquery); + hr = ctx->service->lpVtbl->ExecQuery( ctx->service, - L"WQL", - wquery, + bstrLanguage, + bstrQuery, WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, - NULL, + ctx->wmi_context, &enumerator); + SysFreeString(bstrLanguage); + SysFreeString(bstrQuery); + flb_free(wquery); if (FAILED(hr)) { @@ -363,6 +416,12 @@ static int wmi_exec_query_fixed_val(struct flb_we *ctx, struct wmi_query_spec *s &class_obj, &ret); if(0 == ret) { + if (FAILED(hr)) { + flb_plg_error(ctx->ins, "WMI Next() failed for fixed val. Error code = %x", hr); + } + else { + flb_plg_debug(ctx->ins, "WMI Next() returned 0 items for %s", spec->wmi_counter); + } break; } @@ -397,6 +456,12 @@ static int wmi_exec_query(struct flb_we *ctx, struct wmi_query_spec *spec) &class_obj, &ret); if(0 == ret) { + if (FAILED(hr)) { + flb_plg_error(ctx->ins, "WMI Next() failed for query. Error code = %x", hr); + } + else { + flb_plg_debug(ctx->ins, "WMI Next() returned 0 items for %s", spec->wmi_counter); + } break; } @@ -425,6 +490,10 @@ static int wmi_cleanup(struct flb_we *ctx) ctx->locator->lpVtbl->Release(ctx->locator); ctx->locator = NULL; } + if (ctx->wmi_context != NULL) { + ctx->wmi_context->lpVtbl->Release(ctx->wmi_context); + ctx->wmi_context = NULL; + } CoUninitialize(); return 0; @@ -476,6 +545,7 @@ int we_wmi_init(struct flb_we *ctx) { ctx->locator = NULL; ctx->service = NULL; + ctx->wmi_context = NULL; return 0; } diff --git a/plugins/in_winevtlog/in_winevtlog.c b/plugins/in_winevtlog/in_winevtlog.c index 34a83bc1d53..3a57e46e2df 100644 --- a/plugins/in_winevtlog/in_winevtlog.c +++ b/plugins/in_winevtlog/in_winevtlog.c @@ -189,6 +189,24 @@ static int in_winevtlog_init(struct flb_input_instance *in, return -1; } + /* Rendering options are mutually exclusive */ + if (ctx->render_event_as_xml && ctx->render_event_as_text) { + flb_plg_error(in, + "render_event_as_xml and render_event_as_text cannot be enabled at the same time"); + flb_log_event_encoder_destroy(ctx->log_encoder); + flb_free(ctx); + return -1; + } + + if (ctx->render_event_as_text) { + if (ctx->render_event_text_key == NULL || ctx->render_event_text_key[0] == '\0') { + flb_plg_error(in, "render_event_text_key cannot be empty when render_event_as_text is enabled"); + flb_log_event_encoder_destroy(ctx->log_encoder); + flb_free(ctx); + return -1; + } + } + if (ctx->backoff_multiplier_str && ctx->backoff_multiplier_str[0] != '\0') { mult = atof(ctx->backoff_multiplier_str); if (mult <= 0.0) { @@ -464,7 +482,17 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "render_event_as_xml", "false", 0, FLB_TRUE, offsetof(struct winevtlog_config, render_event_as_xml), - "Whether to consume at oldest records in channels" + "Render Windows EventLog as XML (System and Message fields)" + }, + { + FLB_CONFIG_MAP_BOOL, "render_event_as_text", "false", + 0, FLB_TRUE, offsetof(struct winevtlog_config, render_event_as_text), + "Render Windows EventLog as newline-separated key=value text" + }, + { + FLB_CONFIG_MAP_STR, "render_event_text_key", "log", + 0, FLB_TRUE, offsetof(struct winevtlog_config, render_event_text_key), + "Record key name used when render_event_as_text is enabled" }, { FLB_CONFIG_MAP_BOOL, "use_ansi", "false", diff --git a/plugins/in_winevtlog/pack.c b/plugins/in_winevtlog/pack.c index 688f3ae1044..6182151ffea 100644 --- a/plugins/in_winevtlog/pack.c +++ b/plugins/in_winevtlog/pack.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,79 @@ static int pack_wstr(struct winevtlog_config *ctx, const wchar_t *wstr) return pack_str_codepage(ctx, wstr, CP_UTF8, ctx->use_ansi); } +static int wstr_to_utf8(struct winevtlog_config *ctx, const wchar_t *wstr, + char **out_buf, size_t *out_len) +{ + UINT cp = ctx->use_ansi ? CP_ACP : CP_UTF8; + DWORD flags = (cp == CP_UTF8) ? WC_ERR_INVALID_CHARS : 0; + int size; + char *buf; + + if (out_buf == NULL || out_len == NULL) { + return -1; + } + + *out_buf = NULL; + *out_len = 0; + + if (wstr == NULL) { + return 0; + } + + size = WideCharToMultiByte(cp, flags, wstr, -1, NULL, 0, NULL, NULL); + if (size == 0) { + return -1; + } + + buf = flb_malloc(size); + if (buf == NULL) { + flb_errno(); + return -1; + } + + if (WideCharToMultiByte(cp, flags, wstr, -1, buf, size, NULL, NULL) == 0) { + flb_free(buf); + return -1; + } + + *out_buf = buf; + *out_len = (size_t) (size - 1); + + return 0; +} + +static int append_kv_line(flb_sds_t *text, const char *key, + const char *val, size_t val_len) +{ + if (text == NULL || *text == NULL || key == NULL) { + return -1; + } + + *text = flb_sds_cat(*text, key, strlen(key)); + if (*text == NULL) { + return -1; + } + + *text = flb_sds_cat(*text, "=", 1); + if (*text == NULL) { + return -1; + } + + if (val != NULL && val_len > 0) { + *text = flb_sds_cat(*text, val, val_len); + if (*text == NULL) { + return -1; + } + } + + *text = flb_sds_cat(*text, "\n", 1); + if (*text == NULL) { + return -1; + } + + return 0; +} + static int pack_astr(struct winevtlog_config *ctx, const char *astr) { wchar_t *wbuf = NULL; @@ -336,12 +410,229 @@ static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) return 0; } +static int filetime_to_string(ULONGLONG filetime, char **out_buf, size_t *out_len) +{ + FILETIME ft; + SYSTEMTIME st_utc; + SYSTEMTIME st_local; + DYNAMIC_TIME_ZONE_INFORMATION dtzi; + CHAR buf[64]; + size_t len; + LONG bias_minutes; + int offset_hours; + int offset_minutes; + char offset_sign; + DWORD tz_id; + char *out; + + if (out_buf == NULL || out_len == NULL) { + return -1; + } + + *out_buf = NULL; + *out_len = 0; + + _tzset(); + + ft.dwHighDateTime = (DWORD)(filetime >> 32); + ft.dwLowDateTime = (DWORD)(filetime & 0xFFFFFFFF); + + if (!FileTimeToSystemTime(&ft, &st_utc)) { + return -1; + } + + tz_id = GetDynamicTimeZoneInformation(&dtzi); + + if (!SystemTimeToTzSpecificLocalTimeEx(&dtzi, &st_utc, &st_local)) { + return -1; + } + + bias_minutes = dtzi.Bias; + + if (tz_id == TIME_ZONE_ID_DAYLIGHT) { + bias_minutes += dtzi.DaylightBias; + } + else if (tz_id == TIME_ZONE_ID_STANDARD) { + bias_minutes += dtzi.StandardBias; + } + + if (bias_minutes > 0) { + offset_sign = '-'; + } + else { + offset_sign = '+'; + bias_minutes = -bias_minutes; + } + + offset_hours = bias_minutes / 60; + offset_minutes = bias_minutes % 60; + + len = _snprintf_s(buf, sizeof(buf), _TRUNCATE, + "%04d-%02d-%02d %02d:%02d:%02d %c%02d%02d", + st_local.wYear, + st_local.wMonth, + st_local.wDay, + st_local.wHour, + st_local.wMinute, + st_local.wSecond, + offset_sign, + offset_hours, + offset_minutes); + + if ((int) len <= 0) { + return -1; + } + + out = flb_malloc(len + 1); + if (out == NULL) { + flb_errno(); + return -1; + } + + memcpy(out, buf, len); + out[len] = '\0'; + + *out_buf = out; + *out_len = len; + + return 0; +} + +static int guid_to_utf8(struct winevtlog_config *ctx, const GUID *guid, + char **out_buf, size_t *out_len) +{ + LPOLESTR wguid = NULL; + int ret; + + if (out_buf == NULL || out_len == NULL) { + return -1; + } + + *out_buf = NULL; + *out_len = 0; + + if (guid == NULL) { + return 0; + } + + if (FAILED(StringFromCLSID(guid, &wguid))) { + return -1; + } + + ret = wstr_to_utf8(ctx, wguid, out_buf, out_len); + CoTaskMemFree(wguid); + + return ret; +} + +static int sid_to_utf8(struct winevtlog_config *ctx, PSID sid, + char **out_buf, size_t *out_len) +{ +#define MAX_NAME 256 + LPWSTR wide_sid = NULL; + DWORD acct_len = MAX_NAME, domain_len = MAX_NAME; + DWORD err = ERROR_SUCCESS; + SID_NAME_USE sid_type = SidTypeUnknown; + char account[MAX_NAME]; + char domain[MAX_NAME]; + char formatted[(MAX_NAME * 2) + 2]; + size_t formatted_len; + char *out; + + if (out_buf == NULL || out_len == NULL) { + return -1; + } + + *out_buf = NULL; + *out_len = 0; + + if (sid == NULL) { + return 0; + } + + if (!ConvertSidToStringSidW(sid, &wide_sid)) { + return -1; + } + + /* Skip friendly-name resolution for capability SIDs */ + if (wcsnicmp(wide_sid, L"S-1-15-3-", 9) != 0) { + if (LookupAccountSidA(NULL, sid, + account, &acct_len, domain, + &domain_len, &sid_type)) { + _snprintf_s(formatted, sizeof(formatted), _TRUNCATE, "%s\\%s", domain, account); + formatted_len = strlen(formatted); + if (formatted_len > 0) { + out = flb_malloc(formatted_len + 1); + if (out == NULL) { + flb_errno(); + LocalFree(wide_sid); + return -1; + } + memcpy(out, formatted, formatted_len + 1); + *out_buf = out; + *out_len = formatted_len; + LocalFree(wide_sid); + return 0; + } + } + else { + err = GetLastError(); + if (err != ERROR_NONE_MAPPED) { + flb_plg_debug(ctx->ins, "LookupAccountSidA failed with error code (%u)", err); + } + } + } + + /* Fallback to SID string */ + if (wstr_to_utf8(ctx, wide_sid, out_buf, out_len) != 0) { + LocalFree(wide_sid); + return -1; + } + + LocalFree(wide_sid); + return 0; +#undef MAX_NAME +} + +static int uint_to_string_u64(uint64_t val, char **out_buf, size_t *out_len) +{ + char buf[32]; + int len; + char *out; + + if (out_buf == NULL || out_len == NULL) { + return -1; + } + + *out_buf = NULL; + *out_len = 0; + + len = _snprintf_s(buf, sizeof(buf), _TRUNCATE, "%llu", (unsigned long long) val); + if (len <= 0) { + return -1; + } + + out = flb_malloc((size_t) len + 1); + if (out == NULL) { + flb_errno(); + return -1; + } + memcpy(out, buf, (size_t) len); + out[len] = '\0'; + + *out_buf = out; + *out_len = (size_t) len; + + return 0; +} + static int pack_sid(struct winevtlog_config *ctx, PSID sid, int extract_sid) { #define MAX_NAME 256 size_t size; LPWSTR wide_sid = NULL; - DWORD len = MAX_NAME, err = ERROR_SUCCESS; + DWORD acct_len = MAX_NAME, domain_len = MAX_NAME; + DWORD err = ERROR_SUCCESS; int ret = -1; SID_NAME_USE sid_type = SidTypeUnknown; char account[MAX_NAME]; @@ -362,8 +653,8 @@ static int pack_sid(struct winevtlog_config *ctx, PSID sid, int extract_sid) goto not_mapped_error; } if (!LookupAccountSidA(NULL, sid, - account, &len, domain, - &len, &sid_type)) { + account, &acct_len, domain, + &domain_len, &sid_type)) { err = GetLastError(); if (err == ERROR_NONE_MAPPED) { flb_plg_debug(ctx->ins, "AccountSid is not mapped. code: %u", err); @@ -583,6 +874,250 @@ void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, } } +void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, + PEVT_VARIANT string_inserts, UINT count_inserts, + struct winevtlog_channel *ch, struct winevtlog_config *ctx) +{ + int ret; + flb_sds_t text; + size_t out_len; + char *tmp = NULL; + size_t tmp_len = 0; + char numbuf[64]; + int n; + + (void) ch; + + text = flb_sds_create_size(512); + if (text == NULL) { + return; + } + + /* ProviderName */ + if (wstr_to_utf8(ctx, system[EvtSystemProviderName].StringVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "ProviderName", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "ProviderName", NULL, 0); + } + + /* ProviderGuid */ + tmp = NULL; + tmp_len = 0; + if (system[EvtSystemProviderGuid].Type != EvtVarTypeNull && + guid_to_utf8(ctx, system[EvtSystemProviderGuid].GuidVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "ProviderGuid", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "ProviderGuid", NULL, 0); + } + + /* Qualifiers */ + if (system[EvtSystemQualifiers].Type != EvtVarTypeNull) { + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", (unsigned int) system[EvtSystemQualifiers].UInt16Val); + append_kv_line(&text, "Qualifiers", numbuf, (size_t) (n > 0 ? n : 0)); + } + else { + append_kv_line(&text, "Qualifiers", NULL, 0); + } + + /* EventID */ + if (system[EvtSystemEventID].Type != EvtVarTypeNull) { + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", (unsigned int) system[EvtSystemEventID].UInt16Val); + append_kv_line(&text, "EventID", numbuf, (size_t) (n > 0 ? n : 0)); + } + else { + append_kv_line(&text, "EventID", NULL, 0); + } + + /* Version */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", + (unsigned int) ((system[EvtSystemVersion].Type != EvtVarTypeNull) ? system[EvtSystemVersion].ByteVal : 0)); + append_kv_line(&text, "Version", numbuf, (size_t) (n > 0 ? n : 0)); + + /* Level */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", + (unsigned int) ((system[EvtSystemLevel].Type != EvtVarTypeNull) ? system[EvtSystemLevel].ByteVal : 0)); + append_kv_line(&text, "Level", numbuf, (size_t) (n > 0 ? n : 0)); + + /* Task */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", + (unsigned int) ((system[EvtSystemTask].Type != EvtVarTypeNull) ? system[EvtSystemTask].UInt16Val : 0)); + append_kv_line(&text, "Task", numbuf, (size_t) (n > 0 ? n : 0)); + + /* Opcode */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%u", + (unsigned int) ((system[EvtSystemOpcode].Type != EvtVarTypeNull) ? system[EvtSystemOpcode].ByteVal : 0)); + append_kv_line(&text, "Opcode", numbuf, (size_t) (n > 0 ? n : 0)); + + /* Keywords */ + if (system[EvtSystemKeywords].Type != EvtVarTypeNull) { + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "0x%llx", + (unsigned long long) system[EvtSystemKeywords].UInt64Val); + append_kv_line(&text, "Keywords", numbuf, (size_t) (n > 0 ? n : 0)); + } + else { + append_kv_line(&text, "Keywords", "0", 1); + } + + /* TimeCreated */ + tmp = NULL; + tmp_len = 0; + if (filetime_to_string(system[EvtSystemTimeCreated].FileTimeVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "TimeCreated", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "TimeCreated", NULL, 0); + } + + /* EventRecordID */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%llu", + (unsigned long long) ((system[EvtSystemEventRecordId].Type != EvtVarTypeNull) ? + system[EvtSystemEventRecordId].UInt64Val : 0)); + append_kv_line(&text, "EventRecordID", numbuf, (size_t) (n > 0 ? n : 0)); + + /* ActivityID */ + tmp = NULL; + tmp_len = 0; + if (system[EvtSystemActivityID].Type != EvtVarTypeNull && + guid_to_utf8(ctx, system[EvtSystemActivityID].GuidVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "ActivityID", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "ActivityID", NULL, 0); + } + + /* RelatedActivityID */ + tmp = NULL; + tmp_len = 0; + if (system[EvtSystemRelatedActivityID].Type != EvtVarTypeNull && + guid_to_utf8(ctx, system[EvtSystemRelatedActivityID].GuidVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "RelatedActivityID", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "RelatedActivityID", NULL, 0); + } + + /* ProcessID */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%lu", + (unsigned long) ((system[EvtSystemProcessID].Type != EvtVarTypeNull) ? system[EvtSystemProcessID].UInt32Val : 0)); + append_kv_line(&text, "ProcessID", numbuf, (size_t) (n > 0 ? n : 0)); + + /* ThreadID */ + n = _snprintf_s(numbuf, sizeof(numbuf), _TRUNCATE, "%lu", + (unsigned long) ((system[EvtSystemThreadID].Type != EvtVarTypeNull) ? system[EvtSystemThreadID].UInt32Val : 0)); + append_kv_line(&text, "ThreadID", numbuf, (size_t) (n > 0 ? n : 0)); + + /* Channel */ + tmp = NULL; + tmp_len = 0; + if (wstr_to_utf8(ctx, system[EvtSystemChannel].StringVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "Channel", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "Channel", NULL, 0); + } + + /* Computer */ + tmp = NULL; + tmp_len = 0; + if (wstr_to_utf8(ctx, system[EvtSystemComputer].StringVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "Computer", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "Computer", NULL, 0); + } + + /* UserID */ + tmp = NULL; + tmp_len = 0; + if (sid_to_utf8(ctx, system[EvtSystemUserID].SidVal, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "UserID", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "UserID", NULL, 0); + } + + /* Message */ + tmp = NULL; + tmp_len = 0; + if (wstr_to_utf8(ctx, message, &tmp, &tmp_len) == 0) { + append_kv_line(&text, "Message", tmp, tmp_len); + if (tmp) { + flb_free(tmp); + } + } + else { + append_kv_line(&text, "Message", NULL, 0); + } + + if (text == NULL) { + return; + } + + out_len = flb_sds_len(text); + if (out_len > 0 && text[out_len - 1] == '\n') { + out_len -= 1; + } + + ret = flb_log_event_encoder_begin_record(ctx->log_encoder); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_string(ctx->log_encoder, + ctx->render_event_text_key, + flb_sds_len(ctx->render_event_text_key)); + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_append_body_string(ctx->log_encoder, text, out_len); + } + + /* StringInserts must not be embedded in the key=value text payload. When both + * render_event_as_text and string_inserts are enabled, we expose inserts as a + * structured field under the "StringInserts" key to preserve record-level + * fidelity and avoid mixing formats in TextFormat output. + */ + if (ret == FLB_EVENT_ENCODER_SUCCESS && ctx->string_inserts) { + ret = flb_log_event_encoder_append_body_cstring(ctx->log_encoder, "StringInserts"); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + pack_string_inserts(ctx, string_inserts, count_inserts); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_commit_record(ctx->log_encoder); + } + + flb_sds_destroy(text); +} + void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, struct winevtlog_config *ctx) diff --git a/plugins/in_winevtlog/winevtlog.c b/plugins/in_winevtlog/winevtlog.c index df9f4358a05..8063228c1e2 100644 --- a/plugins/in_winevtlog/winevtlog.c +++ b/plugins/in_winevtlog/winevtlog.c @@ -920,11 +920,31 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, read_size += (system_size + message_size + string_inserts_size); winevtlog_pack_xml_event(system_xml, message, string_inserts, count_inserts, ch, ctx); + } + + flb_free(string_inserts); + flb_free(system_xml); + if (message) { + flb_free(message); + } + } + else if (ctx->render_event_as_text) { + rendered_system = NULL; + render_system_event(ch->events[i], &rendered_system, &system_size); + message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); + get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); + if (rendered_system) { + /* Caluculate total allocated size: system + message + string_inserts */ + read_size += (system_size + message_size + string_inserts_size); + winevtlog_pack_text_event(rendered_system, message, string_inserts, + count_inserts, ch, ctx); - flb_free(string_inserts); - flb_free(system_xml); - if (message) - flb_free(message); + } + + flb_free(string_inserts); + flb_free(rendered_system); + if (message) { + flb_free(message); } } else { @@ -936,11 +956,12 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, read_size += (system_size + message_size + string_inserts_size); winevtlog_pack_event(rendered_system, message, string_inserts, count_inserts, ch, ctx); + } - flb_free(string_inserts); - flb_free(rendered_system); - if (message) - flb_free(message); + flb_free(string_inserts); + flb_free(rendered_system); + if (message) { + flb_free(message); } } } diff --git a/plugins/in_winevtlog/winevtlog.h b/plugins/in_winevtlog/winevtlog.h index f2ea19f0480..8fcb0fc273c 100644 --- a/plugins/in_winevtlog/winevtlog.h +++ b/plugins/in_winevtlog/winevtlog.h @@ -43,8 +43,10 @@ struct winevtlog_config { int string_inserts; int read_existing_events; int render_event_as_xml; + int render_event_as_text; int use_ansi; int ignore_missing_channels; + flb_sds_t render_event_text_key; flb_sds_t event_query; flb_sds_t remote_server; flb_sds_t remote_domain; @@ -133,6 +135,9 @@ void winevtlog_close_all(struct mk_list *list); void winevtlog_pack_xml_event(WCHAR *system_xml, WCHAR *message, PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, struct winevtlog_config *ctx); +void winevtlog_pack_text_event(PEVT_VARIANT system, WCHAR *message, + PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, + struct winevtlog_config *ctx); void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, PEVT_VARIANT string_inserts, UINT count_inserts, struct winevtlog_channel *ch, struct winevtlog_config *ctx); diff --git a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c index c72559f9ad6..7d89ef0075e 100644 --- a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c +++ b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c @@ -374,6 +374,11 @@ static void cb_azure_logs_ingestion_flush(struct flb_event_chunk *event_chunk, static int cb_azure_logs_ingestion_exit(void *data, struct flb_config *config) { struct flb_az_li *ctx = data; + + if (!ctx) { + return 0; + } + flb_plg_debug(ctx->ins, "exiting logs ingestion plugin"); flb_az_li_ctx_destroy(ctx); return 0; diff --git a/plugins/out_chronicle/chronicle.c b/plugins/out_chronicle/chronicle.c index 41ee15e5bcd..030a17b65ba 100644 --- a/plugins/out_chronicle/chronicle.c +++ b/plugins/out_chronicle/chronicle.c @@ -674,6 +674,222 @@ struct chronicle_entry { struct cfl_list _head; }; +static void chronicle_entries_destroy(struct cfl_list *entry_list) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct chronicle_entry *entry; + + cfl_list_foreach_safe(head, tmp, entry_list) { + entry = cfl_list_entry(head, struct chronicle_entry, _head); + flb_sds_destroy(entry->log_text); + cfl_list_del(&entry->_head); + flb_free(entry); + } +} + +struct chronicle_resolved_label { + flb_sds_t key; + flb_sds_t value; + struct mk_list _head; +}; + +static void chronicle_resolved_labels_destroy(struct mk_list *labels) +{ + struct mk_list *head; + struct mk_list *tmp; + struct chronicle_resolved_label *label; + + mk_list_foreach_safe(head, tmp, labels) { + label = mk_list_entry(head, struct chronicle_resolved_label, _head); + mk_list_del(&label->_head); + flb_sds_destroy(label->key); + flb_sds_destroy(label->value); + flb_free(label); + } +} + +static int chronicle_resolved_label_add(struct mk_list *labels, + flb_sds_t key, flb_sds_t value) +{ + struct chronicle_resolved_label *label; + + label = flb_calloc(1, sizeof(struct chronicle_resolved_label)); + if (!label) { + flb_errno(); + return -1; + } + + label->key = flb_sds_create(key); + if (!label->key) { + flb_free(label); + return -1; + } + + label->value = flb_sds_create(value); + if (!label->value) { + flb_sds_destroy(label->key); + flb_free(label); + return -1; + } + + mk_list_add(&label->_head, labels); + return 0; +} + +static int chronicle_sds_equal(flb_sds_t left, flb_sds_t right) +{ + size_t left_len; + size_t right_len; + + if (left == NULL && right == NULL) { + return FLB_TRUE; + } + + if (left == NULL || right == NULL) { + return FLB_FALSE; + } + + left_len = flb_sds_len(left); + right_len = flb_sds_len(right); + + if (left_len != right_len) { + return FLB_FALSE; + } + + if (strncmp(left, right, left_len) != 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static int chronicle_resolved_labels_match(struct mk_list *left, + struct mk_list *right) +{ + struct mk_list *left_head; + struct mk_list *right_head; + struct chronicle_resolved_label *left_label; + struct chronicle_resolved_label *right_label; + + if (mk_list_size(left) != mk_list_size(right)) { + return FLB_FALSE; + } + + right_head = right->next; + mk_list_foreach(left_head, left) { + if (right_head == right) { + return FLB_FALSE; + } + + left_label = mk_list_entry(left_head, struct chronicle_resolved_label, _head); + right_label = mk_list_entry(right_head, struct chronicle_resolved_label, _head); + + if (chronicle_sds_equal(left_label->key, right_label->key) == FLB_FALSE || + chronicle_sds_equal(left_label->value, right_label->value) == FLB_FALSE) { + return FLB_FALSE; + } + + right_head = right_head->next; + } + + return FLB_TRUE; +} + +static void chronicle_resolved_labels_move(struct mk_list *dst, + struct mk_list *src) +{ + struct mk_list *head; + struct mk_list *tmp; + + mk_list_foreach_safe(head, tmp, src) { + mk_list_del(head); + mk_list_add(head, dst); + } +} + +static int chronicle_metadata_match(flb_sds_t left_namespace, + struct mk_list *left_labels, + flb_sds_t right_namespace, + struct mk_list *right_labels) +{ + if (chronicle_sds_equal(left_namespace, right_namespace) == FLB_FALSE) { + return FLB_FALSE; + } + + return chronicle_resolved_labels_match(left_labels, right_labels); +} + +static int chronicle_metadata_resolve(struct flb_chronicle *ctx, + char *tag, int tag_len, + msgpack_object map, + flb_sds_t *namespace, + struct mk_list *labels, + int *label_count) +{ + int ret; + struct mk_list *head; + struct flb_chronicle_label *label; + flb_sds_t value; + + *label_count = 0; + + if (ctx->namespace_ra) { + *namespace = flb_ra_translate_check(ctx->namespace_ra, + tag, tag_len, + map, NULL, FLB_TRUE); + if (!*namespace || flb_sds_len(*namespace) == 0) { + if (*namespace) { + flb_sds_destroy(*namespace); + *namespace = NULL; + } + if (ctx->namespace) { + *namespace = flb_sds_create(ctx->namespace); + } + else { + flb_plg_warn(ctx->ins, + "namespace_key '%s' did not resolve for this batch", + ctx->namespace_key); + } + } + } + else if (ctx->namespace) { + *namespace = flb_sds_create(ctx->namespace); + } + + mk_list_foreach(head, &ctx->labels) { + label = mk_list_entry(head, struct flb_chronicle_label, _head); + + if (label->ra) { + value = flb_ra_translate_check(label->ra, + tag, tag_len, + map, NULL, FLB_TRUE); + if (!value) { + flb_plg_warn(ctx->ins, + "label '%s' did not resolve for this batch", + label->key); + continue; + } + } + else { + value = label->value; + } + + ret = chronicle_resolved_label_add(labels, label->key, value); + if (label->ra) { + flb_sds_destroy(value); + } + + if (ret != 0) { + return -1; + } + + (*label_count)++; + } + + return 0; +} + static int chronicle_format(const void *data, size_t bytes, const char *tag, size_t tag_len, char **out_data, size_t *out_size, @@ -687,7 +903,7 @@ static int chronicle_format(const void *data, size_t bytes, int ret; int array_size = 0; size_t off = 0; - size_t last_off = 0; + size_t last_off = last_offset; size_t alloc_size = 0; size_t s; char time_formatted[255]; @@ -702,10 +918,32 @@ static int chronicle_format(const void *data, size_t bytes, char *json_str; struct cfl_list entry_list; struct chronicle_entry *entry; - struct cfl_list *tmp; struct cfl_list *head; + struct mk_list resolved_labels; + struct mk_list record_labels; + struct mk_list *label_head; + struct chronicle_resolved_label *label; + flb_sds_t namespace = NULL; + flb_sds_t record_namespace = NULL; + int label_count = 0; + int record_label_count = 0; + int map_size = 3; + int metadata_resolved = FLB_FALSE; + size_t record_start; + + if (out_data) { + *out_data = NULL; + } + if (out_size) { + *out_size = 0; + } + if (out_offset) { + *out_offset = last_offset; + } cfl_list_init(&entry_list); + mk_list_init(&resolved_labels); + mk_list_init(&record_labels); array_size = count_mp_with_threshold(last_offset, threshold, log_decoder, ctx); @@ -717,13 +955,14 @@ static int chronicle_format(const void *data, size_t bytes, if (last_offset != 0) { log_decoder->offset = last_offset; } + off = last_offset; while ((ret = flb_log_event_decoder_next( log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { + record_start = off; off = log_decoder->offset; - alloc_size = (off - last_off) + 128; /* JSON is larger than msgpack */ - last_off = off; + alloc_size = (off - record_start) + 128; /* JSON is larger than msgpack */ if (ctx->log_key != NULL) { log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event, config); @@ -757,8 +996,49 @@ static int chronicle_format(const void *data, size_t bytes, return -1; } + mk_list_init(&record_labels); + record_namespace = NULL; + record_label_count = 0; + ret = chronicle_metadata_resolve(ctx, + (char *) tag, tag_len, + *log_event.body, + &record_namespace, + &record_labels, + &record_label_count); + if (ret != 0) { + flb_sds_destroy(log_text); + chronicle_resolved_labels_destroy(&record_labels); + flb_sds_destroy(record_namespace); + chronicle_resolved_labels_destroy(&resolved_labels); + flb_sds_destroy(namespace); + chronicle_entries_destroy(&entry_list); + return -1; + } + + if (metadata_resolved == FLB_FALSE) { + namespace = record_namespace; + record_namespace = NULL; + chronicle_resolved_labels_move(&resolved_labels, &record_labels); + label_count = record_label_count; + metadata_resolved = FLB_TRUE; + } + else if (chronicle_metadata_match(namespace, &resolved_labels, + record_namespace, &record_labels) == FLB_FALSE) { + flb_plg_debug(ctx->ins, + "splitting Chronicle batch due to namespace or label change"); + flb_sds_destroy(log_text); + chronicle_resolved_labels_destroy(&record_labels); + flb_sds_destroy(record_namespace); + last_off = record_start; + break; + } + + chronicle_resolved_labels_destroy(&record_labels); + flb_sds_destroy(record_namespace); + entry = flb_malloc(sizeof(struct chronicle_entry)); if (!entry) { + flb_sds_destroy(log_text); continue; } @@ -767,6 +1047,7 @@ static int chronicle_format(const void *data, size_t bytes, entry->timestamp = log_event.timestamp; cfl_list_add(&entry->_head, &entry_list); + last_off = off; if (off >= (threshold + last_offset)) { flb_plg_debug(ctx->ins, @@ -780,6 +1061,8 @@ static int chronicle_format(const void *data, size_t bytes, /* If the list is empty, no records were valid. */ if (cfl_list_is_empty(&entry_list)) { + chronicle_resolved_labels_destroy(&resolved_labels); + flb_sds_destroy(namespace); return -1; } @@ -808,7 +1091,15 @@ static int chronicle_format(const void *data, size_t bytes, * ] * } */ - msgpack_pack_map(&mp_pck, 3); + if (namespace) { + map_size++; + } + + if (label_count > 0) { + map_size++; + } + + msgpack_pack_map(&mp_pck, map_size); msgpack_pack_str(&mp_pck, 11); msgpack_pack_str_body(&mp_pck, "customer_id", 11); @@ -822,6 +1113,37 @@ static int chronicle_format(const void *data, size_t bytes, msgpack_pack_str(&mp_pck, strlen(ctx->log_type)); msgpack_pack_str_body(&mp_pck, ctx->log_type, strlen(ctx->log_type)); + if (namespace) { + msgpack_pack_str(&mp_pck, 9); + msgpack_pack_str_body(&mp_pck, "namespace", 9); + + msgpack_pack_str(&mp_pck, flb_sds_len(namespace)); + msgpack_pack_str_body(&mp_pck, namespace, flb_sds_len(namespace)); + } + + if (label_count > 0) { + msgpack_pack_str(&mp_pck, 6); + msgpack_pack_str_body(&mp_pck, "labels", 6); + + msgpack_pack_array(&mp_pck, label_count); + + mk_list_foreach(label_head, &resolved_labels) { + label = mk_list_entry(label_head, struct chronicle_resolved_label, _head); + + msgpack_pack_map(&mp_pck, 2); + + msgpack_pack_str(&mp_pck, 3); + msgpack_pack_str_body(&mp_pck, "key", 3); + msgpack_pack_str(&mp_pck, flb_sds_len(label->key)); + msgpack_pack_str_body(&mp_pck, label->key, flb_sds_len(label->key)); + + msgpack_pack_str(&mp_pck, 5); + msgpack_pack_str_body(&mp_pck, "value", 5); + msgpack_pack_str(&mp_pck, flb_sds_len(label->value)); + msgpack_pack_str_body(&mp_pck, label->value, flb_sds_len(label->value)); + } + } + msgpack_pack_str(&mp_pck, 7); msgpack_pack_str_body(&mp_pck, "entries", 7); @@ -829,7 +1151,7 @@ static int chronicle_format(const void *data, size_t bytes, msgpack_pack_array(&mp_pck, cfl_list_size(&entry_list)); /* Iterate the list of valid entries and pack them */ - cfl_list_foreach_safe(head, tmp, &entry_list) { + cfl_list_foreach(head, &entry_list) { entry = cfl_list_entry(head, struct chronicle_entry, _head); /* * Pack entries @@ -863,16 +1185,15 @@ static int chronicle_format(const void *data, size_t bytes, msgpack_pack_str(&mp_pck, s); msgpack_pack_str_body(&mp_pck, time_formatted, s); - /* Clean up the entry now that it's packed */ - flb_sds_destroy(entry->log_text); - cfl_list_del(&entry->_head); - flb_free(entry); } /* Convert from msgpack to JSON */ out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); + chronicle_resolved_labels_destroy(&resolved_labels); + flb_sds_destroy(namespace); + chronicle_entries_destroy(&entry_list); if (!out_buf) { flb_plg_error(ctx->ins, "error formatting JSON payload"); @@ -1162,6 +1483,21 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_chronicle, log_key), "Set the log key" }, + { + FLB_CONFIG_MAP_STR, "namespace", NULL, + 0, FLB_TRUE, offsetof(struct flb_chronicle, namespace), + "Set the Chronicle namespace" + }, + { + FLB_CONFIG_MAP_STR, "namespace_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_chronicle, namespace_key), + "Record accessor for the Chronicle namespace. Falls back to namespace" + }, + { + FLB_CONFIG_MAP_SLIST_1, "label", NULL, + FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_chronicle, label_properties), + "Add a Chronicle label key/value pair. Multiple labels can be set" + }, /* EOF */ {0} }; diff --git a/plugins/out_chronicle/chronicle.h b/plugins/out_chronicle/chronicle.h index 1f2c42e0f61..7e6bcbc346f 100644 --- a/plugins/out_chronicle/chronicle.h +++ b/plugins/out_chronicle/chronicle.h @@ -23,6 +23,7 @@ #include #include #include +#include #include /* refresh token every 50 minutes */ @@ -56,6 +57,13 @@ struct flb_chronicle_oauth_credentials { flb_sds_t token_uri; }; +struct flb_chronicle_label { + flb_sds_t key; + flb_sds_t value; + struct flb_record_accessor *ra; + struct mk_list _head; +}; + struct flb_chronicle { /* credentials */ flb_sds_t credentials_file; @@ -72,6 +80,11 @@ struct flb_chronicle { flb_sds_t endpoint; flb_sds_t region; flb_sds_t log_key; + flb_sds_t namespace; + flb_sds_t namespace_key; + struct flb_record_accessor *namespace_ra; + struct mk_list *label_properties; + struct mk_list labels; int json_date_format; flb_sds_t json_date_key; diff --git a/plugins/out_chronicle/chronicle_conf.c b/plugins/out_chronicle/chronicle_conf.c index 9b851247f23..b0c3f222a58 100644 --- a/plugins/out_chronicle/chronicle_conf.c +++ b/plugins/out_chronicle/chronicle_conf.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -40,6 +41,105 @@ static inline int key_cmp(char *str, int len, char *cmp) { return strncasecmp(str, cmp, len); } +static void chronicle_labels_destroy(struct flb_chronicle *ctx) +{ + struct mk_list *head; + struct mk_list *tmp; + struct flb_chronicle_label *label; + + mk_list_foreach_safe(head, tmp, &ctx->labels) { + label = mk_list_entry(head, struct flb_chronicle_label, _head); + mk_list_del(&label->_head); + + flb_sds_destroy(label->key); + flb_sds_destroy(label->value); + + if (label->ra) { + flb_ra_destroy(label->ra); + } + + flb_free(label); + } +} + +static int chronicle_label_add(struct flb_chronicle *ctx, + char *key, size_t key_len, + char *value, size_t value_len) +{ + struct flb_chronicle_label *label; + + if (key == NULL || key_len == 0 || value == NULL) { + flb_plg_error(ctx->ins, "label requires a non-empty key and value"); + return -1; + } + + label = flb_calloc(1, sizeof(struct flb_chronicle_label)); + if (!label) { + flb_errno(); + return -1; + } + + label->key = flb_sds_create_len(key, key_len); + if (!label->key) { + flb_free(label); + return -1; + } + + label->value = flb_sds_create_len(value, value_len); + if (!label->value) { + flb_sds_destroy(label->key); + flb_free(label); + return -1; + } + + if (memchr(value, '$', value_len) != NULL) { + label->ra = flb_ra_create(label->value, FLB_FALSE); + if (!label->ra) { + flb_plg_error(ctx->ins, "invalid label record accessor '%s'", + label->value); + flb_sds_destroy(label->key); + flb_sds_destroy(label->value); + flb_free(label); + return -1; + } + } + + mk_list_add(&label->_head, &ctx->labels); + return 0; +} + +static int chronicle_configure_labels(struct flb_chronicle *ctx) +{ + int ret; + struct mk_list *head; + struct flb_config_map_val *mv; + struct flb_slist_entry *key; + struct flb_slist_entry *val; + + if (ctx->label_properties) { + flb_config_map_foreach(head, mv, ctx->label_properties) { + if (mk_list_size(mv->val.list) != 2) { + flb_plg_error(ctx->ins, "'label' expects a key and a value"); + return -1; + } + + key = mk_list_entry_first(mv->val.list, + struct flb_slist_entry, _head); + val = mk_list_entry_last(mv->val.list, + struct flb_slist_entry, _head); + + ret = chronicle_label_add(ctx, + key->str, flb_sds_len(key->str), + val->str, flb_sds_len(val->str)); + if (ret != 0) { + return -1; + } + } + } + + return 0; +} + static int flb_chronicle_read_credentials_file(struct flb_chronicle *ctx, char *creds, struct flb_chronicle_oauth_credentials *ctx_creds) @@ -187,6 +287,7 @@ struct flb_chronicle *flb_chronicle_conf_create(struct flb_output_instance *ins, } ctx->ins = ins; ctx->config = config; + mk_list_init(&ctx->labels); ret = flb_output_config_map_set(ins, (void *)ctx); if (ret == -1) { @@ -302,6 +403,28 @@ struct flb_chronicle *flb_chronicle_conf_create(struct flb_output_instance *ins, return NULL; } + if (ctx->namespace && flb_sds_len(ctx->namespace) == 0) { + flb_plg_error(ctx->ins, "property 'namespace' cannot be empty"); + flb_chronicle_conf_destroy(ctx); + return NULL; + } + + if (ctx->namespace_key) { + ctx->namespace_ra = flb_ra_create(ctx->namespace_key, FLB_FALSE); + if (!ctx->namespace_ra) { + flb_plg_error(ctx->ins, "invalid namespace_key record accessor '%s'", + ctx->namespace_key); + flb_chronicle_conf_destroy(ctx); + return NULL; + } + } + + ret = chronicle_configure_labels(ctx); + if (ret != 0) { + flb_chronicle_conf_destroy(ctx); + return NULL; + } + /* Date key */ ctx->date_key = ctx->json_date_key; tmp = flb_output_get_property("json_date_key", ins); @@ -418,6 +541,12 @@ int flb_chronicle_conf_destroy(struct flb_chronicle *ctx) flb_oauth2_destroy(ctx->o); } + if (ctx->namespace_ra) { + flb_ra_destroy(ctx->namespace_ra); + } + + chronicle_labels_destroy(ctx); + flb_free(ctx); return 0; } diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c index 459da80cbe5..bd724c587d4 100644 --- a/plugins/out_es/es.c +++ b/plugins/out_es/es.c @@ -235,6 +235,21 @@ static flb_sds_t es_get_id_value(struct flb_elasticsearch *ctx, return tmp_str; } +static int es_action_line_value_is_safe(const char *value, size_t len) +{ + size_t i; + unsigned char c; + + for (i = 0; i < len; i++) { + c = (unsigned char) value[i]; + if (c == '\n' || c == '\r' || c == '"' || c == '\\' || c < 0x20) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + static int compose_index_header(struct flb_elasticsearch *ctx, int es_index_custom_len, char *logstash_index, size_t logstash_index_size, @@ -293,6 +308,10 @@ static int elasticsearch_format(struct flb_config *config, int len; int map_size; int index_len = 0; + int write_op_update = FLB_FALSE; + int write_op_upsert = FLB_FALSE; + int id_key_required = FLB_FALSE; + int id_key_safe; size_t s = 0; size_t off = 0; size_t off_prev = 0; @@ -345,10 +364,16 @@ static int elasticsearch_format(struct flb_config *config, return -1; } - /* Copy logstash prefix if logstash format is enabled */ - if (ctx->logstash_format == FLB_TRUE) { - strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); - logstash_index[sizeof(logstash_index) - 1] = '\0'; + if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPDATE) == 0) { + write_op_update = FLB_TRUE; + } + else if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPSERT) == 0) { + write_op_upsert = FLB_TRUE; + } + + if (ctx->ra_id_key && ctx->generate_id == FLB_FALSE && + (write_op_update == FLB_TRUE || write_op_upsert == FLB_TRUE)) { + id_key_required = FLB_TRUE; } /* @@ -403,6 +428,12 @@ static int elasticsearch_format(struct flb_config *config, map = *log_event.body; map_size = map.via.map.size; + /* Copy logstash prefix for the per-record fallback path. */ + if (ctx->logstash_format == FLB_TRUE) { + strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); + logstash_index[sizeof(logstash_index) - 1] = '\0'; + } + es_index_custom_len = 0; if (ctx->logstash_prefix_key) { flb_sds_t v = flb_ra_translate(ctx->ra_prefix_key, @@ -410,14 +441,16 @@ static int elasticsearch_format(struct flb_config *config, map, NULL); if (v) { len = flb_sds_len(v); - if (len > 128) { - len = 128; - memcpy(logstash_index, v, 128); - } - else { - memcpy(logstash_index, v, len); + if (es_action_line_value_is_safe(v, len) == FLB_TRUE) { + if (len > 128) { + len = 128; + memcpy(logstash_index, v, 128); + } + else { + memcpy(logstash_index, v, len); + } + es_index_custom_len = len; } - es_index_custom_len = len; flb_sds_destroy(v); } } @@ -538,7 +571,15 @@ static int elasticsearch_format(struct flb_config *config, } if (ctx->ra_id_key) { id_key_str = es_get_id_value(ctx ,&map); - if (id_key_str) { + id_key_safe = FLB_FALSE; + + if (id_key_str && + es_action_line_value_is_safe(id_key_str, + flb_sds_len(id_key_str)) == FLB_TRUE) { + id_key_safe = FLB_TRUE; + } + + if (id_key_safe == FLB_TRUE) { if (ctx->suppress_type_name) { index_len = flb_sds_snprintf(&j_index, flb_sds_alloc(j_index), @@ -553,6 +594,35 @@ static int elasticsearch_format(struct flb_config *config, ctx->es_action, es_index, ctx->type, id_key_str); } + } + else if (id_key_required == FLB_TRUE) { + flb_plg_warn(ctx->ins, + "skipping record with missing or unsafe Id_Key value"); + if (id_key_str) { + flb_sds_destroy(id_key_str); + id_key_str = NULL; + } + msgpack_sbuffer_destroy(&tmp_sbuf); + continue; + } + else if (ctx->generate_id == FLB_FALSE && id_key_str) { + if (ctx->suppress_type_name) { + index_len = flb_sds_snprintf(&j_index, + flb_sds_alloc(j_index), + ES_BULK_INDEX_FMT_WITHOUT_TYPE, + ctx->es_action, + es_index); + } + else { + index_len = flb_sds_snprintf(&j_index, + flb_sds_alloc(j_index), + ES_BULK_INDEX_FMT, + ctx->es_action, + es_index, ctx->type); + } + } + + if (id_key_str) { flb_sds_destroy(id_key_str); id_key_str = NULL; } @@ -570,13 +640,13 @@ static int elasticsearch_format(struct flb_config *config, } out_buf_len = flb_sds_len(out_buf); - if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPDATE) == 0) { + if (write_op_update == FLB_TRUE) { tmp_buf = out_buf; out_buf = flb_sds_create_len(NULL, out_buf_len = out_buf_len + sizeof(ES_BULK_UPDATE_OP_BODY) - 2); out_buf_len = snprintf(out_buf, out_buf_len, ES_BULK_UPDATE_OP_BODY, tmp_buf); flb_sds_destroy(tmp_buf); } - else if (strcasecmp(ctx->write_operation, FLB_ES_WRITE_OP_UPSERT) == 0) { + else if (write_op_upsert == FLB_TRUE) { tmp_buf = out_buf; out_buf = flb_sds_create_len(NULL, out_buf_len = out_buf_len + sizeof(ES_BULK_UPSERT_OP_BODY) - 2); out_buf_len = snprintf(out_buf, out_buf_len, ES_BULK_UPSERT_OP_BODY, tmp_buf); @@ -845,6 +915,12 @@ static void cb_es_flush(struct flb_event_chunk *event_chunk, FLB_OUTPUT_RETURN(FLB_ERROR); } + if (out_size == 0) { + flb_free(out_buf); + flb_upstream_conn_release(u_conn); + FLB_OUTPUT_RETURN(FLB_OK); + } + pack = (char *) out_buf; pack_size = out_size; diff --git a/plugins/out_forward/forward.c b/plugins/out_forward/forward.c index a73b95cfe60..992743df159 100644 --- a/plugins/out_forward/forward.c +++ b/plugins/out_forward/forward.c @@ -339,22 +339,39 @@ static void secure_forward_set_ping(struct flb_forward_ping *ping, memset(ping, 0, sizeof(struct flb_forward_ping)); ping->keepalive = 1; /* default, as per spec */ + if (map->type != MSGPACK_OBJECT_MAP) { + return; + } + for (i = 0; i < map->via.map.size; i++) { key = map->via.map.ptr[i].key; val = map->via.map.ptr[i].val; + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + ptr = key.via.str.ptr; len = key.via.str.size; if (len == 5 && memcmp(ptr, "nonce", len) == 0) { + if (val.type != MSGPACK_OBJECT_STR && val.type != MSGPACK_OBJECT_BIN) { + continue; + } ping->nonce = val.via.bin.ptr; ping->nonce_len = val.via.bin.size; } else if (len == 4 && memcmp(ptr, "auth", len) == 0) { + if (val.type != MSGPACK_OBJECT_STR && val.type != MSGPACK_OBJECT_BIN) { + continue; + } ping->auth = val.via.bin.ptr; ping->auth_len = val.via.bin.size; } else if (len == 9 && memcmp(ptr, "keepalive", len) == 0) { + if (val.type != MSGPACK_OBJECT_BOOLEAN) { + continue; + } ping->keepalive = val.via.boolean; } } @@ -548,7 +565,7 @@ static int secure_forward_pong(struct flb_forward *ctx, char *buf, int buf_size) goto error; } - if (strncmp(o.via.str.ptr, "PONG", 4) != 0 || o.via.str.size != 4) { + if (o.via.str.size != 4 || strncmp(o.via.str.ptr, "PONG", 4) != 0) { goto error; } @@ -563,7 +580,17 @@ static int secure_forward_pong(struct flb_forward *ctx, char *buf, int buf_size) } else { o = root.via.array.ptr[2]; - memcpy(msg, o.via.str.ptr, o.via.str.size); + if (o.type != MSGPACK_OBJECT_STR) { + goto error; + } + if (o.via.str.size >= sizeof(msg)) { + memcpy(msg, o.via.str.ptr, sizeof(msg) - 1); + msg[sizeof(msg) - 1] = '\0'; + } + else { + memcpy(msg, o.via.str.ptr, o.via.str.size); + msg[o.via.str.size] = '\0'; + } flb_plg_error(ctx->ins, "failed authorization: %s", msg); } @@ -602,6 +629,12 @@ static int secure_forward_handshake(struct flb_connection *u_conn, /* Parse HELO message */ root = result.data; + if (root.type != MSGPACK_OBJECT_ARRAY) { + flb_plg_error(ctx->ins, "Invalid HELO type message"); + msgpack_unpacked_destroy(&result); + return -1; + } + if (root.via.array.size < 2) { flb_plg_error(ctx->ins, "Invalid HELO message"); msgpack_unpacked_destroy(&result); @@ -615,7 +648,7 @@ static int secure_forward_handshake(struct flb_connection *u_conn, return -1; } - if (strncmp(o.via.str.ptr, "HELO", 4) != 0 || o.via.str.size != 4) { + if (o.via.str.size != 4 || strncmp(o.via.str.ptr, "HELO", 4) != 0) { flb_plg_error(ctx->ins, "Invalid HELO content message"); msgpack_unpacked_destroy(&result); return -1; @@ -625,6 +658,12 @@ static int secure_forward_handshake(struct flb_connection *u_conn, /* Compose and send PING message */ o = root.via.array.ptr[1]; + if (o.type != MSGPACK_OBJECT_MAP) { + flb_plg_error(ctx->ins, "Invalid HELO options message"); + msgpack_unpacked_destroy(&result); + return -1; + } + ret = secure_forward_ping(u_conn, o, fc, ctx); if (ret == -1) { flb_plg_error(ctx->ins, "Failed PING"); @@ -699,8 +738,16 @@ static int forward_read_ack(struct flb_forward *ctx, /* Lookup ack field */ for (i = 0; i < map.size; i++) { key = map.ptr[i].key; + if (key.type != MSGPACK_OBJECT_STR) { + continue; + } + if (key.via.str.size == 3 && strncmp(key.via.str.ptr, "ack", 3) == 0) { val = map.ptr[i].val; + if (val.type != MSGPACK_OBJECT_STR) { + goto error; + } + ack_len = val.via.str.size; ack = val.via.str.ptr; break; @@ -714,15 +761,15 @@ static int forward_read_ack(struct flb_forward *ctx, if (ack_len != chunk_len) { flb_plg_error(ctx->ins, - "ack: ack len does not match ack(%ld)(%.*s) chunk(%d)(%.*s)", + "ack: ack len does not match ack(%zu)(%.*s) chunk(%d)(%.*s)", ack_len, (int) ack_len, ack, chunk_len, (int) chunk_len, chunk); goto error; } if (strncmp(ack, chunk, ack_len) != 0) { - flb_plg_error(ctx->ins, "ACK: mismatch received=%s, expected=(%.*s)", - ack, chunk_len, chunk); + flb_plg_error(ctx->ins, "ACK: mismatch received=%.*s, expected=(%.*s)", + (int) ack_len, ack, chunk_len, chunk); goto error; } diff --git a/plugins/out_influxdb/influxdb.c b/plugins/out_influxdb/influxdb.c index 2e73d43276a..9fce46bf4b6 100644 --- a/plugins/out_influxdb/influxdb.c +++ b/plugins/out_influxdb/influxdb.c @@ -686,7 +686,7 @@ static struct flb_config_map config_map[] = { }, { - FLB_CONFIG_MAP_BOOL, "tag_keys", NULL, + FLB_CONFIG_MAP_STR, "tag_keys", NULL, 0, FLB_FALSE, 0, "Space separated list of keys that needs to be tagged." }, diff --git a/plugins/out_kafka/kafka.c b/plugins/out_kafka/kafka.c index 1be9cd4ed6b..f4e8965bc86 100644 --- a/plugins/out_kafka/kafka.c +++ b/plugins/out_kafka/kafka.c @@ -141,7 +141,7 @@ int produce_message(struct flb_time *tm, msgpack_object *map, // an embedded schemaid which is used // the embedding is a null byte // followed by a 16 byte schemaid -#define AVRO_SCHEMA_OVERHEAD 16 + 1 +#define AVRO_SCHEMA_OVERHEAD 4 + 1 #endif flb_debug("in produce_message\n"); @@ -312,29 +312,29 @@ int produce_message(struct flb_time *tm, msgpack_object *map, #ifdef FLB_HAVE_AVRO_ENCODER else if (ctx->format == FLB_KAFKA_FMT_AVRO) { - flb_plg_debug(ctx->ins, "avro schema ID:%s:\n", ctx->avro_fields.schema_id); + flb_plg_debug(ctx->ins, "avro schema ID:%d:\n", ctx->avro_fields.schema_id); flb_plg_debug(ctx->ins, "avro schema string:%s:\n", ctx->avro_fields.schema_str); // if there's no data then log it and return if (mp_sbuf.size == 0) { - flb_plg_error(ctx->ins, "got zero bytes decoding to avro AVRO:schemaID:%s:\n", ctx->avro_fields.schema_id); + flb_plg_error(ctx->ins, "got zero bytes decoding to avro AVRO:schemaID:%d:\n", ctx->avro_fields.schema_id); msgpack_sbuffer_destroy(&mp_sbuf); return FLB_OK; } // is the line is too long log it and return if (mp_sbuf.size > AVRO_LINE_MAX_LEN) { - flb_plg_warn(ctx->ins, "skipping long line AVRO:len:%zu:limit:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, (size_t)AVRO_LINE_MAX_LEN, ctx->avro_fields.schema_id); + flb_plg_warn(ctx->ins, "skipping long line AVRO:len:%zu:limit:%zu:schemaID:%d:\n", (size_t)mp_sbuf.size, (size_t)AVRO_LINE_MAX_LEN, ctx->avro_fields.schema_id); msgpack_sbuffer_destroy(&mp_sbuf); return FLB_OK; } - flb_plg_debug(ctx->ins, "using default buffer AVRO:len:%zu:limit:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, (size_t)AVRO_DEFAULT_BUFFER_SIZE, ctx->avro_fields.schema_id); + flb_plg_debug(ctx->ins, "using default buffer AVRO:len:%zu:limit:%zu:schemaID:%d:\n", (size_t)mp_sbuf.size, (size_t)AVRO_DEFAULT_BUFFER_SIZE, ctx->avro_fields.schema_id); out_buf = avro_buff; out_size = AVRO_DEFAULT_BUFFER_SIZE; if (mp_sbuf.size + AVRO_SCHEMA_OVERHEAD >= AVRO_DEFAULT_BUFFER_SIZE) { - flb_plg_info(ctx->ins, "upsizing to dynamic buffer AVRO:len:%zu:schemaID:%s:\n", (size_t)mp_sbuf.size, ctx->avro_fields.schema_id); + flb_plg_info(ctx->ins, "upsizing to dynamic buffer AVRO:len:%zu:schemaID:%d:\n", (size_t)mp_sbuf.size, ctx->avro_fields.schema_id); avro_fast_buffer = false; // avro will always be smaller than msgpack // it contains no meta-info aside from the schemaid @@ -344,14 +344,14 @@ int produce_message(struct flb_time *tm, msgpack_object *map, out_size = mp_sbuf.size + AVRO_SCHEMA_OVERHEAD; out_buf = flb_malloc(out_size); if (!out_buf) { - flb_plg_error(ctx->ins, "error allocating memory for decoding to AVRO:schema:%s:schemaID:%s:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); + flb_plg_error(ctx->ins, "error allocating memory for decoding to AVRO:schema:%s:schemaID:%d:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); msgpack_sbuffer_destroy(&mp_sbuf); return FLB_ERROR; } } if(!flb_msgpack_raw_to_avro_sds(mp_sbuf.data, mp_sbuf.size, &ctx->avro_fields, out_buf, &out_size)) { - flb_plg_error(ctx->ins, "error encoding to AVRO:schema:%s:schemaID:%s:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); + flb_plg_error(ctx->ins, "error encoding to AVRO:schema:%s:schemaID:%d:\n", ctx->avro_fields.schema_str, ctx->avro_fields.schema_id); msgpack_sbuffer_destroy(&mp_sbuf); if (!avro_fast_buffer) { flb_free(out_buf); @@ -639,8 +639,8 @@ static struct flb_config_map config_map[] = { "Set AVRO schema." }, { - FLB_CONFIG_MAP_STR, "schema_id", (char *)NULL, - 0, FLB_FALSE, 0, + FLB_CONFIG_MAP_INT, "schema_id", (char *)NULL, + 0, FLB_TRUE, offsetof(struct flb_out_kafka, avro_fields) + offsetof(struct flb_avro_fields, schema_id), "Set AVRO schema ID." }, #endif diff --git a/plugins/out_kafka/kafka_config.c b/plugins/out_kafka/kafka_config.c index b1f07458884..91e52e4323c 100644 --- a/plugins/out_kafka/kafka_config.c +++ b/plugins/out_kafka/kafka_config.c @@ -249,10 +249,6 @@ struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, if (tmp) { ctx->avro_fields.schema_str = flb_sds_create(tmp); } - tmp = flb_output_get_property("schema_id", ins); - if (tmp) { - ctx->avro_fields.schema_id = flb_sds_create(tmp); - } #endif /* Config: Topic */ @@ -282,7 +278,7 @@ struct flb_out_kafka *flb_out_kafka_create(struct flb_output_instance *ins, flb_plg_info(ctx->ins, "brokers='%s' topics='%s'", ctx->kafka.brokers, tmp); #ifdef FLB_HAVE_AVRO_ENCODER - flb_plg_info(ctx->ins, "schemaID='%s' schema='%s'", ctx->avro_fields.schema_id, ctx->avro_fields.schema_str); + flb_plg_info(ctx->ins, "schemaID='%d' schema='%s'", ctx->avro_fields.schema_id, ctx->avro_fields.schema_str); #endif return ctx; @@ -324,7 +320,6 @@ int flb_out_kafka_destroy(struct flb_out_kafka *ctx) #ifdef FLB_HAVE_AVRO_ENCODER // avro - flb_sds_destroy(ctx->avro_fields.schema_id); flb_sds_destroy(ctx->avro_fields.schema_str); #endif diff --git a/plugins/out_logdna/logdna.c b/plugins/out_logdna/logdna.c index d594dbc7234..7c79bb9ffdb 100644 --- a/plugins/out_logdna/logdna.c +++ b/plugins/out_logdna/logdna.c @@ -26,6 +26,12 @@ #include "logdna.h" +#define LOGDNA_META_KEY "meta" +#define LOGDNA_LEVEL_KEY "level" +#define LOGDNA_SEVERITY_KEY "severity" +#define LOGDNA_FILE_KEY "file" +#define LOGDNA_APP_KEY "app" + static inline int primary_key_check(msgpack_object k, char *name, int len) { if (k.type != MSGPACK_OBJECT_STR) { @@ -44,19 +50,26 @@ static inline int primary_key_check(msgpack_object k, char *name, int len) } /* - * This function looks for the following keys and add them to the buffer + * This function looks for the following primary keys and promotes them to + * the top-level line object: * * - level or severity * - file * - app * - meta + * + * When line_pck is not NULL, non-primary keys are packed into it for use + * as the "line" body (excluding the promoted keys). */ static int record_append_primary_keys(struct flb_logdna *ctx, msgpack_object *map, - msgpack_packer *mp_sbuf) + msgpack_packer *mp_sbuf, + msgpack_packer *line_pck) { int i; int c = 0; + int is_primary; + int line_count = 0; msgpack_object *level = NULL; msgpack_object *file = NULL; msgpack_object *app = NULL; @@ -64,63 +77,94 @@ static int record_append_primary_keys(struct flb_logdna *ctx, msgpack_object k; msgpack_object v; - for (i = 0; i < map->via.array.size; i++) { + if (line_pck) { + for (i = 0; i < map->via.map.size; i++) { + k = map->via.map.ptr[i].key; + if (primary_key_check(k, LOGDNA_META_KEY, sizeof(LOGDNA_META_KEY) - 1) == FLB_TRUE || + primary_key_check(k, LOGDNA_LEVEL_KEY, sizeof(LOGDNA_LEVEL_KEY) - 1) == FLB_TRUE || + primary_key_check(k, LOGDNA_SEVERITY_KEY, sizeof(LOGDNA_SEVERITY_KEY) - 1) == FLB_TRUE || + primary_key_check(k, LOGDNA_FILE_KEY, sizeof(LOGDNA_FILE_KEY) - 1) == FLB_TRUE || + primary_key_check(k, LOGDNA_APP_KEY, sizeof(LOGDNA_APP_KEY) - 1) == FLB_TRUE) { + continue; + } + line_count++; + } + msgpack_pack_map(line_pck, line_count); + } + + for (i = 0; i < map->via.map.size; i++) { k = map->via.map.ptr[i].key; v = map->via.map.ptr[i].val; - - /* Level - optional */ - if (!level && - (primary_key_check(k, "level", 5) == FLB_TRUE || - primary_key_check(k, "severity", 8) == FLB_TRUE)) { - level = &k; - msgpack_pack_str(mp_sbuf, 5); - msgpack_pack_str_body(mp_sbuf, "level", 5); - msgpack_pack_object(mp_sbuf, v); - c++; + is_primary = FLB_FALSE; + + /* Level - optional (both "level" and "severity" are primary) */ + if (primary_key_check(k, LOGDNA_LEVEL_KEY, sizeof(LOGDNA_LEVEL_KEY) - 1) == FLB_TRUE || + primary_key_check(k, LOGDNA_SEVERITY_KEY, sizeof(LOGDNA_SEVERITY_KEY) - 1) == FLB_TRUE) { + is_primary = FLB_TRUE; + if (!level) { + level = &k; + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_LEVEL_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_LEVEL_KEY, sizeof(LOGDNA_LEVEL_KEY) - 1); + msgpack_pack_object(mp_sbuf, v); + c++; + } } /* Meta - optional */ - if (!meta && primary_key_check(k, "meta", 4) == FLB_TRUE) { - meta = &k; - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "meta", 4); - msgpack_pack_object(mp_sbuf, v); - c++; + if (primary_key_check(k, LOGDNA_META_KEY, sizeof(LOGDNA_META_KEY) - 1) == FLB_TRUE) { + is_primary = FLB_TRUE; + if (!meta) { + meta = &k; + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_META_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_META_KEY, sizeof(LOGDNA_META_KEY) - 1); + msgpack_pack_object(mp_sbuf, v); + c++; + } } /* File */ - if (!file && primary_key_check(k, "file", 4) == FLB_TRUE) { - file = &k; - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "file", 4); - msgpack_pack_object(mp_sbuf, v); - c++; + if (primary_key_check(k, LOGDNA_FILE_KEY, sizeof(LOGDNA_FILE_KEY) - 1) == FLB_TRUE) { + is_primary = FLB_TRUE; + if (!file) { + file = &k; + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_FILE_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_FILE_KEY, sizeof(LOGDNA_FILE_KEY) - 1); + msgpack_pack_object(mp_sbuf, v); + c++; + } } /* App */ - if (primary_key_check(k, "app", 3) == FLB_TRUE) { - app = &k; - msgpack_pack_str(mp_sbuf, 3); - msgpack_pack_str_body(mp_sbuf, "app", 3); - msgpack_pack_object(mp_sbuf, v); - c++; + if (primary_key_check(k, LOGDNA_APP_KEY, sizeof(LOGDNA_APP_KEY) - 1) == FLB_TRUE) { + is_primary = FLB_TRUE; + if (!app) { + app = &k; + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_APP_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_APP_KEY, sizeof(LOGDNA_APP_KEY) - 1); + msgpack_pack_object(mp_sbuf, v); + c++; + } + } + + if (line_pck && is_primary == FLB_FALSE) { + msgpack_pack_object(line_pck, k); + msgpack_pack_object(line_pck, v); } } /* Set the global file name if the record did not provided one */ if (!file && ctx->file) { - msgpack_pack_str(mp_sbuf, 4); - msgpack_pack_str_body(mp_sbuf, "file", 4); + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_FILE_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_FILE_KEY, sizeof(LOGDNA_FILE_KEY) - 1); msgpack_pack_str(mp_sbuf, flb_sds_len(ctx->file)); msgpack_pack_str_body(mp_sbuf, ctx->file, flb_sds_len(ctx->file)); c++; } - /* If no application name is set, set the default */ if (!app) { - msgpack_pack_str(mp_sbuf, 3); - msgpack_pack_str_body(mp_sbuf, "app", 3); + msgpack_pack_str(mp_sbuf, sizeof(LOGDNA_APP_KEY) - 1); + msgpack_pack_str_body(mp_sbuf, LOGDNA_APP_KEY, sizeof(LOGDNA_APP_KEY) - 1); msgpack_pack_str(mp_sbuf, flb_sds_len(ctx->app)); msgpack_pack_str_body(mp_sbuf, ctx->app, flb_sds_len(ctx->app)); c++; @@ -139,10 +183,14 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, int total_lines; int array_size = 0; off_t map_off; + size_t off; char *line_json; flb_sds_t json; msgpack_packer mp_pck; msgpack_sbuffer mp_sbuf; + msgpack_packer mp_line_pck; + msgpack_sbuffer mp_line_sbuf; + msgpack_unpacked mp_line_result; struct flb_log_event_decoder log_decoder; struct flb_log_event log_event; @@ -178,10 +226,24 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, msgpack_pack_map(&mp_pck, array_size); /* - * Append primary keys found, the return values is the number of appended + * Append primary keys found, the return value is the number of appended * keys to the record, we use that to adjust the map header size. + * + * When exclude_promoted_keys is enabled, non-primary keys are packed + * into mp_line_sbuf for use as the "line" body. */ - ret = record_append_primary_keys(ctx, log_event.body, &mp_pck); + if (ctx->exclude_promoted_keys) { + msgpack_sbuffer_init(&mp_line_sbuf); + msgpack_packer_init(&mp_line_pck, &mp_line_sbuf, + msgpack_sbuffer_write); + + ret = record_append_primary_keys(ctx, log_event.body, + &mp_pck, &mp_line_pck); + } + else { + ret = record_append_primary_keys(ctx, log_event.body, + &mp_pck, NULL); + } array_size += ret; /* Timestamp */ @@ -193,7 +255,23 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, msgpack_pack_str(&mp_pck, 4); msgpack_pack_str_body(&mp_pck, "line", 4); - line_json = flb_msgpack_to_json_str(1024, log_event.body, config->json_escape_unicode); + if (ctx->exclude_promoted_keys) { + msgpack_unpacked_init(&mp_line_result); + off = 0; + msgpack_unpack_next(&mp_line_result, + mp_line_sbuf.data, mp_line_sbuf.size, &off); + + line_json = flb_msgpack_to_json_str(1024, &mp_line_result.data, + config->json_escape_unicode); + + msgpack_unpacked_destroy(&mp_line_result); + msgpack_sbuffer_destroy(&mp_line_sbuf); + } + else { + line_json = flb_msgpack_to_json_str(1024, log_event.body, + config->json_escape_unicode); + } + len = strlen(line_json); msgpack_pack_str(&mp_pck, len); msgpack_pack_str_body(&mp_pck, line_json, len); @@ -584,11 +662,39 @@ static struct flb_config_map config_map[] = { "Name of the application generating the data (optional)" }, + { + FLB_CONFIG_MAP_BOOL, "exclude_promoted_keys", "false", + 0, FLB_TRUE, offsetof(struct flb_logdna, exclude_promoted_keys), + "Exclude promoted keys (meta, level, severity (promoted as level), app, file) from the line body" + }, + /* EOF */ {0} }; +static int cb_logdna_format_test(struct flb_config *config, + struct flb_input_instance *ins, + void *plugin_context, + void *flush_ctx, + int event_type, + const char *tag, int tag_len, + const void *data, size_t bytes, + void **out_data, size_t *out_size) +{ + flb_sds_t json; + struct flb_logdna *ctx = plugin_context; + + json = logdna_compose_payload(ctx, data, bytes, tag, tag_len, config); + if (!json) { + return -1; + } + + *out_data = json; + *out_size = flb_sds_len(json); + return 0; +} + /* Plugin reference */ struct flb_output_plugin out_logdna_plugin = { .name = "logdna", @@ -597,5 +703,6 @@ struct flb_output_plugin out_logdna_plugin = { .cb_flush = cb_logdna_flush, .cb_exit = cb_logdna_exit, .config_map = config_map, + .test_formatter.callback = cb_logdna_format_test, .flags = FLB_OUTPUT_NET | FLB_IO_TLS, }; diff --git a/plugins/out_logdna/logdna.h b/plugins/out_logdna/logdna.h index 7ec8e843d14..98d927825db 100644 --- a/plugins/out_logdna/logdna.h +++ b/plugins/out_logdna/logdna.h @@ -41,6 +41,7 @@ struct flb_logdna { flb_sds_t file; flb_sds_t app; struct mk_list *tags; + int exclude_promoted_keys; /* Internal */ flb_sds_t _hostname; diff --git a/plugins/out_opensearch/opensearch.c b/plugins/out_opensearch/opensearch.c index db654be6d73..5541deea0da 100644 --- a/plugins/out_opensearch/opensearch.c +++ b/plugins/out_opensearch/opensearch.c @@ -225,6 +225,21 @@ static flb_sds_t os_get_id_value(struct flb_opensearch *ctx, return tmp_str; } +static int os_action_line_value_is_safe(const char *value, size_t len) +{ + size_t i; + unsigned char c; + + for (i = 0; i < len; i++) { + c = (unsigned char) value[i]; + if (c == '\n' || c == '\r' || c == '"' || c == '\\' || c < 0x20) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + static int compose_index_header(struct flb_opensearch *ctx, int index_custom_len, char *logstash_index, size_t logstash_index_size, @@ -283,6 +298,8 @@ static int opensearch_format(struct flb_config *config, int index_len = 0; int write_op_update = FLB_FALSE; int write_op_upsert = FLB_FALSE; + int id_key_required = FLB_FALSE; + int id_key_safe; flb_sds_t ra_index = NULL; size_t s = 0; char *index = NULL; @@ -330,10 +347,16 @@ static int opensearch_format(struct flb_config *config, return -1; } - /* Copy logstash prefix if logstash format is enabled */ - if (ctx->logstash_format == FLB_TRUE) { - strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); - logstash_index[sizeof(logstash_index) - 1] = '\0'; + if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPDATE) == 0) { + write_op_update = FLB_TRUE; + } + else if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPSERT) == 0) { + write_op_upsert = FLB_TRUE; + } + + if (ctx->ra_id_key && ctx->generate_id == FLB_FALSE && + (write_op_update == FLB_TRUE || write_op_upsert == FLB_TRUE)) { + id_key_required = FLB_TRUE; } /* @@ -393,6 +416,12 @@ static int opensearch_format(struct flb_config *config, map = *log_event.body; map_size = map.via.map.size; + /* Copy logstash prefix for the per-record fallback path. */ + if (ctx->logstash_format == FLB_TRUE) { + strncpy(logstash_index, ctx->logstash_prefix, sizeof(logstash_index)); + logstash_index[sizeof(logstash_index) - 1] = '\0'; + } + index_custom_len = 0; if (ctx->logstash_prefix_key) { flb_sds_t v = flb_ra_translate(ctx->ra_prefix_key, @@ -400,15 +429,17 @@ static int opensearch_format(struct flb_config *config, map, NULL); if (v) { len = flb_sds_len(v); - if (len > 128) { - len = 128; - memcpy(logstash_index, v, 128); - } - else { - memcpy(logstash_index, v, len); - } + if (os_action_line_value_is_safe(v, len) == FLB_TRUE) { + if (len > 128) { + len = 128; + memcpy(logstash_index, v, 128); + } + else { + memcpy(logstash_index, v, len); + } - index_custom_len = len; + index_custom_len = len; + } flb_sds_destroy(v); } } @@ -492,6 +523,11 @@ static int opensearch_format(struct flb_config *config, if (!ra_index) { flb_plg_warn(ctx->ins, "invalid index translation from record accessor pattern, default to static index"); } + else if (os_action_line_value_is_safe(ra_index, + flb_sds_len(ra_index)) == FLB_FALSE) { + flb_sds_destroy(ra_index); + ra_index = NULL; + } else { index = ra_index; } @@ -566,7 +602,15 @@ static int opensearch_format(struct flb_config *config, } if (ctx->ra_id_key) { id_key_str = os_get_id_value(ctx ,&map); - if (id_key_str) { + id_key_safe = FLB_FALSE; + + if (id_key_str && + os_action_line_value_is_safe(id_key_str, + flb_sds_len(id_key_str)) == FLB_TRUE) { + id_key_safe = FLB_TRUE; + } + + if (id_key_safe == FLB_TRUE) { if (ctx->suppress_type_name) { index_len = flb_sds_snprintf(&j_index, flb_sds_alloc(j_index), @@ -581,6 +625,35 @@ static int opensearch_format(struct flb_config *config, ctx->action, index, ctx->type, id_key_str); } + } + else if (id_key_required == FLB_TRUE) { + flb_plg_warn(ctx->ins, + "skipping record with missing or unsafe Id_Key value"); + if (id_key_str) { + flb_sds_destroy(id_key_str); + id_key_str = NULL; + } + msgpack_sbuffer_destroy(&tmp_sbuf); + continue; + } + else if (ctx->generate_id == FLB_FALSE && id_key_str) { + if (ctx->suppress_type_name) { + index_len = flb_sds_snprintf(&j_index, + flb_sds_alloc(j_index), + OS_BULK_INDEX_FMT_NO_TYPE, + ctx->action, + index); + } + else { + index_len = flb_sds_snprintf(&j_index, + flb_sds_alloc(j_index), + OS_BULK_INDEX_FMT, + ctx->action, + index, ctx->type); + } + } + + if (id_key_str) { flb_sds_destroy(id_key_str); id_key_str = NULL; } @@ -613,13 +686,6 @@ static int opensearch_format(struct flb_config *config, return -1; } - if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPDATE) == 0) { - write_op_update = FLB_TRUE; - } - else if (strcasecmp(ctx->write_operation, FLB_OS_WRITE_OP_UPSERT) == 0) { - write_op_upsert = FLB_TRUE; - } - /* UPDATE | UPSERT */ if (write_op_update) { flb_sds_cat_safe(&bulk, @@ -913,6 +979,12 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, FLB_OUTPUT_RETURN(FLB_ERROR); } + if (out_size == 0) { + flb_sds_destroy(out_buf); + flb_upstream_conn_release(u_conn); + FLB_OUTPUT_RETURN(FLB_OK); + } + pack = (char *) out_buf; pack_size = out_size; diff --git a/plugins/out_stackdriver/gce_metadata.c b/plugins/out_stackdriver/gce_metadata.c index fc0afd098a9..a721d8e5f0e 100644 --- a/plugins/out_stackdriver/gce_metadata.c +++ b/plugins/out_stackdriver/gce_metadata.c @@ -33,7 +33,7 @@ static int fetch_metadata(struct flb_stackdriver *ctx, struct flb_upstream *upstream, char *uri, - char *payload) + flb_sds_t *payload) { int ret; int ret_code; @@ -44,15 +44,30 @@ static int fetch_metadata(struct flb_stackdriver *ctx, /* If runtime test mode is enabled, add test data */ if (ctx->ins->test_mode == FLB_TRUE) { if (strcmp(uri, FLB_STD_METADATA_PROJECT_ID_URI) == 0) { - flb_sds_cat(payload, "fluent-bit-test", 15); + flb_sds_t tmp; + tmp = flb_sds_cat(*payload, "fluent-bit-test", 15); + if (!tmp) { + return -1; + } + *payload = tmp; return 0; } else if (strcmp(uri, FLB_STD_METADATA_ZONE_URI) == 0) { - flb_sds_cat(payload, "projects/0123456789/zones/fluent", 32); + flb_sds_t tmp; + tmp = flb_sds_cat(*payload, "projects/0123456789/zones/fluent", 32); + if (!tmp) { + return -1; + } + *payload = tmp; return 0; } else if (strcmp(uri, FLB_STD_METADATA_INSTANCE_ID_URI) == 0) { - flb_sds_cat(payload, "333222111", 9); + flb_sds_t tmp; + tmp = flb_sds_cat(*payload, "333222111", 9); + if (!tmp) { + return -1; + } + *payload = tmp; return 0; } return -1; @@ -87,8 +102,15 @@ static int fetch_metadata(struct flb_stackdriver *ctx, /* The request was issued successfully, validate the 'error' field */ flb_plg_debug(ctx->ins, "HTTP Status=%i", c->resp.status); if (c->resp.status == 200) { - ret_code = 0; - flb_sds_copy(payload, c->resp.payload, c->resp.payload_size); + flb_sds_t tmp; + tmp = flb_sds_copy(*payload, c->resp.payload, c->resp.payload_size); + if (!tmp) { + ret_code = -1; + } + else { + *payload = tmp; + ret_code = 0; + } } else { if (c->resp.payload_size > 0) { @@ -117,7 +139,7 @@ int gce_metadata_read_token(struct flb_stackdriver *ctx) uri = flb_sds_cat(uri, ctx->client_email, flb_sds_len(ctx->client_email)); uri = flb_sds_cat(uri, "/token", 6); - ret = fetch_metadata(ctx, ctx->metadata_u, uri, payload); + ret = fetch_metadata(ctx, ctx->metadata_u, uri, &payload); if (ret != 0) { flb_plg_error(ctx->ins, "can't fetch token from the metadata server"); flb_sds_destroy(payload); @@ -147,7 +169,7 @@ int gce_metadata_read_zone(struct flb_stackdriver *ctx) flb_sds_t zone = NULL; ret = fetch_metadata(ctx, ctx->metadata_u, FLB_STD_METADATA_ZONE_URI, - payload); + &payload); if (ret != 0) { flb_plg_error(ctx->ins, "can't fetch zone from the metadata server"); flb_sds_destroy(payload); @@ -193,7 +215,7 @@ int gce_metadata_read_project_id(struct flb_stackdriver *ctx) flb_sds_t payload = flb_sds_create_size(4096); ret = fetch_metadata(ctx, ctx->metadata_u, - FLB_STD_METADATA_PROJECT_ID_URI, payload); + FLB_STD_METADATA_PROJECT_ID_URI, &payload); if (ret != 0) { flb_plg_error(ctx->ins, "can't fetch project id from the metadata server"); flb_sds_destroy(payload); @@ -210,7 +232,7 @@ int gce_metadata_read_instance_id(struct flb_stackdriver *ctx) flb_sds_t payload = flb_sds_create_size(4096); ret = fetch_metadata(ctx, ctx->metadata_u, - FLB_STD_METADATA_INSTANCE_ID_URI, payload); + FLB_STD_METADATA_INSTANCE_ID_URI, &payload); if (ret != 0) { flb_plg_error(ctx->ins, "can't fetch instance id from the metadata server"); flb_sds_destroy(payload); diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index b492f2225d8..5aa2568e203 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -50,6 +50,10 @@ pthread_key_t oauth2_type; pthread_key_t oauth2_token; pthread_key_t oauth2_token_expires; +static int oauth2_cache_initialized = FLB_FALSE; +static int oauth2_cache_users = 0; +static pthread_mutex_t oauth2_cache_lock = PTHREAD_MUTEX_INITIALIZER; + static void oauth2_cache_exit(void *ptr) { if (ptr) { @@ -64,12 +68,60 @@ static void oauth2_cache_free_expiration(void *ptr) } } -static void oauth2_cache_init() +static int oauth2_cache_init() +{ + int ret; + + ret = 0; + + pthread_mutex_lock(&oauth2_cache_lock); + + if (oauth2_cache_initialized == FLB_FALSE) { + /* oauth2 pthread key */ + ret = pthread_key_create(&oauth2_type, oauth2_cache_exit); + if (ret != 0) { + goto done; + } + ret = pthread_key_create(&oauth2_token, oauth2_cache_exit); + if (ret != 0) { + pthread_key_delete(oauth2_type); + goto done; + } + ret = pthread_key_create(&oauth2_token_expires, + oauth2_cache_free_expiration); + if (ret != 0) { + pthread_key_delete(oauth2_type); + pthread_key_delete(oauth2_token); + goto done; + } + oauth2_cache_initialized = FLB_TRUE; + } + + if (ret == 0) { + oauth2_cache_users++; + } + +done: + pthread_mutex_unlock(&oauth2_cache_lock); + return ret; +} + +static void oauth2_cache_cleanup(void) { - /* oauth2 pthread key */ - pthread_key_create(&oauth2_type, oauth2_cache_exit); - pthread_key_create(&oauth2_token, oauth2_cache_exit); - pthread_key_create(&oauth2_token_expires, oauth2_cache_free_expiration); + pthread_mutex_lock(&oauth2_cache_lock); + + if (oauth2_cache_users > 0) { + oauth2_cache_users--; + if (oauth2_cache_users == 0 && + oauth2_cache_initialized == FLB_TRUE) { + pthread_key_delete(oauth2_type); + pthread_key_delete(oauth2_token); + pthread_key_delete(oauth2_token_expires); + oauth2_cache_initialized = FLB_FALSE; + } + } + + pthread_mutex_unlock(&oauth2_cache_lock); } /* Set oauth2 type and token in pthread keys */ @@ -1078,6 +1130,9 @@ static int pack_resource_labels(struct flb_stackdriver *ctx, } else { flb_plg_warn(ctx->ins, "failed to find a corresponding entry for " "resource label entry [%s=%s]", label_kv->key, label_kv->val); + if (rval) { + flb_ra_key_value_destroy(rval); + } } flb_ra_destroy(ra); } else { @@ -1221,7 +1276,7 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, /* Load config map */ ret = flb_output_config_map_set(ins, (void *) ctx); if (ret == -1) { - return -1; + goto error; } /* Set context */ @@ -1237,10 +1292,20 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, } /* Initialize oauth2 cache pthread keys */ - oauth2_cache_init(); + ret = oauth2_cache_init(); + if (ret != 0) { + flb_plg_error(ins, "failed to initialize oauth2 cache"); + goto error; + } + ctx->oauth2_cache_acquired = FLB_TRUE; /* Create mutex for acquiring oauth tokens (they are shared across flush coroutines) */ - pthread_mutex_init(&ctx->token_mutex, NULL); + ret = pthread_mutex_init(&ctx->token_mutex, NULL); + if (ret != 0) { + flb_plg_error(ins, "failed to initialize token mutex"); + goto error; + } + ctx->token_mutex_initialized = FLB_TRUE; /* Create Upstream context for Stackdriver Logging (no oauth2 service) */ ctx->u = flb_upstream_create_url(config, ctx->cloud_logging_write_url, @@ -1253,15 +1318,15 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, if (!ctx->u) { flb_plg_error(ctx->ins, "upstream creation failed"); - return -1; + goto error; } if (!ctx->metadata_u) { flb_plg_error(ctx->ins, "metadata upstream creation failed"); - return -1; + goto error; } if (!ctx->o) { flb_plg_error(ctx->ins, "cannot create oauth2 context"); - return -1; + goto error; } flb_output_upstream_set(ctx->u, ins); @@ -1282,19 +1347,19 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, if (ctx->metadata_server_auth) { ret = gce_metadata_read_project_id(ctx); if (ret == -1) { - return -1; + goto error; } if (ctx->resource_type != RESOURCE_TYPE_GENERIC_NODE && ctx->resource_type != RESOURCE_TYPE_GENERIC_TASK) { ret = gce_metadata_read_zone(ctx); if (ret == -1) { - return -1; + goto error; } ret = gce_metadata_read_instance_id(ctx); if (ret == -1) { - return -1; + goto error; } } } @@ -1302,7 +1367,7 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, /* Validate project_id */ if (!ctx->project_id) { flb_plg_error(ctx->ins, "property 'project_id' is not set"); - return -1; + goto error; } if (!ctx->export_to_project_id) { @@ -1312,10 +1377,22 @@ static int cb_stackdriver_init(struct flb_output_instance *ins, ret = flb_stackdriver_regex_init(ctx); if (ret == -1) { flb_plg_error(ctx->ins, "failed to init stackdriver custom regex"); - return -1; + goto error; } return 0; + +error: + if (ctx->token_mutex_initialized == FLB_TRUE) { + pthread_mutex_destroy(&ctx->token_mutex); + ctx->token_mutex_initialized = FLB_FALSE; + } + if (ctx->oauth2_cache_acquired == FLB_TRUE) { + oauth2_cache_cleanup(); + ctx->oauth2_cache_acquired = FLB_FALSE; + } + flb_stackdriver_conf_destroy(ctx); + return -1; } static int validate_severity_level(severity_t * s, @@ -2393,6 +2470,8 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, flb_sds_destroy(log_name); } + destroy_http_request(&http_request); + flb_log_event_decoder_destroy(&log_decoder); msgpack_sbuffer_destroy(&mp_sbuf); @@ -3087,6 +3166,10 @@ static int cb_stackdriver_exit(void *data, struct flb_config *config) return -1; } + if (ctx->oauth2_cache_acquired == FLB_TRUE) { + oauth2_cache_cleanup(); + ctx->oauth2_cache_acquired = FLB_FALSE; + } flb_stackdriver_conf_destroy(ctx); return 0; } diff --git a/plugins/out_stackdriver/stackdriver.h b/plugins/out_stackdriver/stackdriver.h index dd1536963b9..f54faa9d339 100644 --- a/plugins/out_stackdriver/stackdriver.h +++ b/plugins/out_stackdriver/stackdriver.h @@ -202,8 +202,12 @@ struct flb_stackdriver { /* environment variable settings */ struct flb_stackdriver_env *env; + /* oauth2 cache reference */ + int oauth2_cache_acquired; + /* mutex for acquiring oauth tokens */ pthread_mutex_t token_mutex; + int token_mutex_initialized; /* upstream context for stackdriver write end-point */ struct flb_upstream *u; diff --git a/plugins/out_stackdriver/stackdriver_conf.c b/plugins/out_stackdriver/stackdriver_conf.c index 635752d0c57..103232abeec 100644 --- a/plugins/out_stackdriver/stackdriver_conf.c +++ b/plugins/out_stackdriver/stackdriver_conf.c @@ -134,6 +134,9 @@ static int read_credentials_file(const char *cred_file, struct flb_stackdriver * ctx->creds->type = flb_sds_create_len(val, val_len); } else if (key_cmp(key, key_len, "project_id") == 0) { + if (ctx->project_id) { + flb_sds_destroy(ctx->project_id); + } ctx->project_id = flb_sds_create_len(val, val_len); } else if (key_cmp(key, key_len, "private_key_id") == 0) { @@ -706,6 +709,9 @@ int flb_stackdriver_conf_destroy(struct flb_stackdriver *ctx) flb_kv_release(&ctx->config_labels); flb_kv_release(&ctx->resource_labels_kvs); + if (ctx->token_mutex_initialized) { + pthread_mutex_destroy(&ctx->token_mutex); + } flb_free(ctx); return 0; diff --git a/plugins/processor_content_modifier/cm.h b/plugins/processor_content_modifier/cm.h index 80ff20f068f..c94c01b7703 100644 --- a/plugins/processor_content_modifier/cm.h +++ b/plugins/processor_content_modifier/cm.h @@ -97,6 +97,7 @@ struct content_modifier_ctx { flb_sds_t pattern; /* pattern to create 'regex' context */ flb_sds_t converted_type_str; /* converted_type */ flb_sds_t key; /* target key */ + int key_is_autogenerated; /* key ownership for synthesized OTEL keys */ flb_sds_t value; /* used for any value */ struct flb_regex *regex; /* regular expression context created from 'pattern' */ diff --git a/plugins/processor_content_modifier/cm_config.c b/plugins/processor_content_modifier/cm_config.c index b76638804b3..efd5109f181 100644 --- a/plugins/processor_content_modifier/cm_config.c +++ b/plugins/processor_content_modifier/cm_config.c @@ -22,6 +22,7 @@ #include #include "cm.h" +#include "cm_config.h" static int set_action(struct content_modifier_ctx *ctx) { @@ -146,6 +147,11 @@ static int set_context(struct content_modifier_ctx *ctx) /* check that 'name' is the key set */ if (!ctx->key) { ctx->key = flb_sds_create("name"); + if (!ctx->key) { + flb_errno(); + return -1; + } + ctx->key_is_autogenerated = FLB_TRUE; } else if (strcasecmp(ctx->key, "name") != 0) { flb_plg_error(ctx->ins, "context '%s' requires the name of the key to be 'name', no '%s'", @@ -182,6 +188,11 @@ static int set_context(struct content_modifier_ctx *ctx) /* check that 'version' is the key set */ if (!ctx->key) { ctx->key = flb_sds_create("version"); + if (!ctx->key) { + flb_errno(); + return -1; + } + ctx->key_is_autogenerated = FLB_TRUE; } else if (strcasecmp(ctx->key, "version") != 0) { flb_plg_error(ctx->ins, "context '%s' requires the name of the key to be 'version', no '%s'", @@ -256,6 +267,11 @@ static int set_context(struct content_modifier_ctx *ctx) /* check that 'name' is the key set */ if (!ctx->key) { ctx->key = flb_sds_create("name"); + if (!ctx->key) { + flb_errno(); + return -1; + } + ctx->key_is_autogenerated = FLB_TRUE; } else if (strcasecmp(ctx->key, "name") != 0) { flb_plg_error(ctx->ins, "context '%s' requires the name of the key to be 'name', no '%s'", @@ -356,27 +372,27 @@ struct content_modifier_ctx *cm_config_create(struct flb_processor_instance *ins /* Initialize the config map */ ret = flb_processor_instance_config_map_set(ins, ctx); if (ret == -1) { - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } if (!ctx->action_str) { flb_plg_error(ctx->ins, "no 'action' defined"); - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } /* process the 'action' configuration */ ret = set_action(ctx); if (ret == -1) { - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } /* process the 'context' where the action will be applied */ ret = set_context(ctx); if (ret == -1) { - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } @@ -385,7 +401,7 @@ struct content_modifier_ctx *cm_config_create(struct flb_processor_instance *ins ctx->regex = flb_regex_create(ctx->pattern); if (!ctx->regex) { flb_plg_error(ctx->ins, "invalid regex pattern '%s'", ctx->pattern); - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } } @@ -393,7 +409,7 @@ struct content_modifier_ctx *cm_config_create(struct flb_processor_instance *ins /* Certain actions needs extra configuration, e.g: insert -> requires a key and a value */ ret = check_action_requirements(ctx); if (ret == -1) { - flb_free(ctx); + cm_config_destroy(ctx); return NULL; } return ctx; @@ -401,9 +417,17 @@ struct content_modifier_ctx *cm_config_create(struct flb_processor_instance *ins void cm_config_destroy(struct content_modifier_ctx *ctx) { + if (ctx == NULL) { + return; + } + if (ctx->regex) { flb_regex_destroy(ctx->regex); } + if (ctx->key_is_autogenerated == FLB_TRUE && ctx->key != NULL) { + flb_sds_destroy(ctx->key); + } + flb_free(ctx); } diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index fb7c5ced9e1..4a4c3af5619 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: fluent-bit base: core20 -version: '4.2.3' +version: '4.2.7' summary: High performance logs and stream processor description: | Fluent Bit is a high performance log processor and stream processor for Linux. diff --git a/src/aws/compression/arrow/compress.c b/src/aws/compression/arrow/compress.c index 000d629b553..1cae74c30e6 100644 --- a/src/aws/compression/arrow/compress.c +++ b/src/aws/compression/arrow/compress.c @@ -15,6 +15,20 @@ #include #include +static int choose_block_size(size_t size) +{ + int block_size = 8 * 1024 * 1024; + + while ((size_t) block_size <= size) { + block_size *= 2; + if (block_size > 64 * 1024 * 1024) { + return 64 * 1024 * 1024; + } + } + + return block_size; +} + /* * GArrowTable is the central structure that represents "table" (a.k.a. * data frame). @@ -46,6 +60,10 @@ static GArrowTable* parse_json(uint8_t *json, int size) return NULL; } + g_object_set(options, + "block-size", choose_block_size(size), + NULL); + reader = garrow_json_reader_new(GARROW_INPUT_STREAM(input), options, &error); if (reader == NULL) { g_error_free(error); diff --git a/src/aws/flb_aws_util.c b/src/aws/flb_aws_util.c index a0509c8ce54..ef6db9ec375 100644 --- a/src/aws/flb_aws_util.c +++ b/src/aws/flb_aws_util.c @@ -33,8 +33,10 @@ #include #include -#define AWS_SERVICE_ENDPOINT_FORMAT "%s.%s.amazonaws.com" -#define AWS_SERVICE_ENDPOINT_BASE_LEN 15 +#define AWS_SERVICE_ENDPOINT_FORMAT "%s.%s%s" +#define AWS_SERVICE_ENDPOINT_SUFFIX_COM ".amazonaws.com" +#define AWS_SERVICE_ENDPOINT_SUFFIX_COM_CN ".amazonaws.com.cn" +#define AWS_SERVICE_ENDPOINT_SUFFIX_EU ".amazonaws.eu" #define TAG_PART_DESCRIPTOR "$TAG[%d]" #define TAG_DESCRIPTOR "$TAG" @@ -71,29 +73,30 @@ struct flb_http_client *request_do(struct flb_aws_client *aws_client, size_t dynamic_headers_len); /* - * https://service.region.amazonaws.com(.cn) + * https://service.region.amazonaws.[com(.cn)|eu] */ char *flb_aws_endpoint(char* service, char* region) { char *endpoint = NULL; - size_t len = AWS_SERVICE_ENDPOINT_BASE_LEN; - int is_cn = FLB_FALSE; + const char *domain_suffix = AWS_SERVICE_ENDPOINT_SUFFIX_COM; + size_t len; int bytes; - /* In the China regions, ".cn" is appended to the URL */ - if (strcmp("cn-north-1", region) == 0) { - len += 3; - is_cn = FLB_TRUE; + /* China regions end with amazonaws.com.cn */ + if (strcmp("cn-north-1", region) == 0 || + strcmp("cn-northwest-1", region) == 0) { + domain_suffix = AWS_SERVICE_ENDPOINT_SUFFIX_COM_CN; } - if (strcmp("cn-northwest-1", region) == 0) { - len += 3; - is_cn = FLB_TRUE; + else if (strncmp(region, "eusc-", 5) == 0) { + domain_suffix = AWS_SERVICE_ENDPOINT_SUFFIX_EU; } - len += strlen(service); + len = strlen(service); + len += 1; /* dot between service and region */ len += strlen(region); - len++; /* null byte */ + len += strlen(domain_suffix); + len += 1; /* null byte */ endpoint = flb_calloc(len, sizeof(char)); if (!endpoint) { @@ -101,18 +104,13 @@ char *flb_aws_endpoint(char* service, char* region) return NULL; } - bytes = snprintf(endpoint, len, AWS_SERVICE_ENDPOINT_FORMAT, service, region); - if (bytes < 0) { + bytes = snprintf(endpoint, len, AWS_SERVICE_ENDPOINT_FORMAT, service, region, domain_suffix); + if (bytes < 0 || bytes >= len) { flb_errno(); flb_free(endpoint); return NULL; } - if (is_cn) { - memcpy(endpoint + bytes, ".cn", 3); - endpoint[bytes + 3] = '\0'; - } - return endpoint; } diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index 6e64915a553..916093368d9 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -537,8 +537,10 @@ static char *dirname(char *path) ptr = strrchr(path, '\\'); if (ptr == NULL) { - return path; + /* No directory component */ + return "."; } + *ptr++='\0'; return path; } diff --git a/src/flb_avro.c b/src/flb_avro.c index 57e35926c2a..09cea7ea1ad 100644 --- a/src/flb_avro.c +++ b/src/flb_avro.c @@ -18,6 +18,9 @@ */ #include +#include +#include +#include #include #include @@ -28,14 +31,50 @@ #include #include -static inline int do_avro(bool call, const char *msg) { +static inline int do_avro(bool call, const char *msg) +{ if (call) { - flb_error("%s:\n %s\n", msg, avro_strerror()); - return FLB_FALSE; + flb_error("%s:\n %s\n", msg, avro_strerror()); + return FLB_FALSE; } return FLB_TRUE; } +static int set_avro_integer(avro_value_t *val, int64_t i64, uint64_t u64, int positive) +{ + avro_type_t type; + + type = avro_value_get_type(val); + + if (type == AVRO_INT64) { + if (positive && u64 > INT64_MAX) { + flb_error("positive integer value exceeds Avro long range: %" PRIu64, u64); + return FLB_FALSE; + } + + return do_avro(avro_value_set_long(val, positive ? (int64_t) u64 : i64), + positive ? "failed on posint" : "failed on negint"); + } + + if (type == AVRO_INT32) { + if (positive && u64 > INT32_MAX) { + flb_error("positive integer value exceeds Avro int range: %" PRIu64, u64); + return FLB_FALSE; + } + else if (!positive && (i64 < INT32_MIN || i64 > INT32_MAX)) { + flb_error("negative integer value exceeds Avro int range: %" PRIi64, i64); + return FLB_FALSE; + } + + return do_avro(avro_value_set_int(val, positive ? (int32_t) u64 : (int32_t) i64), + positive ? "failed on posint" : "failed on negint"); + } + + flb_error("integer value cannot be assigned to Avro type %d", type); + + return FLB_FALSE; +} + avro_value_iface_t *flb_avro_init(avro_value_t *aobject, char *json, size_t json_len, avro_schema_t *aschema) { @@ -85,33 +124,20 @@ int msgpack2avro(avro_value_t *val, msgpack_object *o) #if defined(PRIu64) // msgpack_pack_fix_uint64 flb_debug("got a posint: %" PRIu64 "\n", o->via.u64); - ret = do_avro(avro_value_set_int(val, o->via.u64), "failed on posint"); #else - if (o.via.u64 > ULONG_MAX) - flb_warn("over \"%lu\"", ULONG_MAX); - ret = do_avro(avro_value_set_int(val, ULONG_MAX), "failed on posint"); - else - flb_debug("got a posint: %lu\n", (unsigned long)o->via.u64); - ret = do_avro(avro_value_set_int(val, o->via.u64), "failed on posint"); + flb_debug("got a posint: %lu\n", (unsigned long) o->via.u64); #endif + ret = set_avro_integer(val, 0, o->via.u64, FLB_TRUE); break; case MSGPACK_OBJECT_NEGATIVE_INTEGER: #if defined(PRIi64) flb_debug("got a negint: %" PRIi64 "\n", o->via.i64); - ret = do_avro(avro_value_set_int(val, o->via.i64), "failed on negint"); #else - if (o->via.i64 > LONG_MAX) - flb_warn("over +\"%ld\"", LONG_MAX); - ret = do_avro(avro_value_set_int(val, LONG_MAX), "failed on negint"); - else if (o->via.i64 < LONG_MIN) - flb_warn("under -\"%ld\"", LONG_MIN); - ret = do_avro(avro_value_set_int(val, LONG_MIN), "failed on negint"); - else - flb_debug("got a negint: %ld\n", (signed long)o->via.i64); - ret = do_avro(avro_value_set_int(val, o->via.i64), "failed on negint"); + flb_debug("got a negint: %ld\n", (signed long) o->via.i64); #endif + ret = set_avro_integer(val, o->via.i64, 0, FLB_FALSE); break; case MSGPACK_OBJECT_FLOAT32: @@ -207,6 +233,10 @@ int msgpack2avro(avro_value_t *val, msgpack_object *o) ret = flb_msgpack_to_avro(&element, &p->val); flb_sds_destroy(key); + + if (ret == FLB_FALSE) { + goto msg2avro_end; + } } } break; @@ -268,7 +298,7 @@ bool flb_msgpack_raw_to_avro_sds(const void *in_buf, size_t in_size, struct flb_ avro_writer_t awriter; flb_debug("in flb_msgpack_raw_to_avro_sds\n"); - flb_debug("schemaID:%s:\n", ctx->schema_id); + flb_debug("schemaID:%d:\n", ctx->schema_id); flb_debug("schema string:%s:\n", ctx->schema_str); size_t schema_json_len = flb_sds_len(ctx->schema_str); @@ -340,19 +370,26 @@ bool flb_msgpack_raw_to_avro_sds(const void *in_buf, size_t in_size, struct flb_ return false; } - // write the schemaid - // its md5hash of the avro schema - // it looks like this c4b52aaf22429c7f9eb8c30270bc1795 - const char *pos = ctx->schema_id; - unsigned char val[16]; - size_t count; - for (count = 0; count < sizeof val/sizeof *val; count++) { - sscanf(pos, "%2hhx", &val[count]); - pos += 2; + // write the schemaID + if (ctx->schema_id <= 0) { + flb_error("Invalid schema_id=%d (must be > 0)\n", ctx->schema_id); + avro_writer_free(awriter); + avro_value_decref(&aobject); + avro_value_iface_decref(aclass); + avro_schema_decref(aschema); + msgpack_unpacked_destroy(&result); + return false; } - + + int32_t id = ctx->schema_id; + unsigned char val[4]; + val[0] = (id >> 24) & 0xFF; + val[1] = (id >> 16) & 0xFF; + val[2] = (id >> 8) & 0xFF; + val[3] = id & 0xFF; + // write it into a buffer which can be passed to librdkafka - rval = avro_write(awriter, val, 16); + rval = avro_write(awriter, val, 4); if (rval != 0) { flb_error("Unable to write schemaid\n"); avro_writer_free(awriter); @@ -373,9 +410,6 @@ bool flb_msgpack_raw_to_avro_sds(const void *in_buf, size_t in_size, struct flb_ return false; } - // null terminate it - avro_write(awriter, "\0", 1); - flb_debug("before avro_writer_flush\n"); avro_writer_flush(awriter); diff --git a/src/flb_engine.c b/src/flb_engine.c index 9d998b1b702..41e63d55c6d 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -290,7 +290,9 @@ static inline int handle_output_event(uint64_t ts, int retry_seconds; uint32_t type; uint32_t key; + int effective_records; double latency_seconds; + size_t effective_bytes; char *in_name; char *out_name; struct flb_task *task; @@ -340,6 +342,9 @@ static inline int handle_output_event(uint64_t ts, } in_name = (char *) flb_input_name(task->i_ins); out_name = (char *) flb_output_name(ins); + effective_records = task->event_chunk->total_events; + effective_bytes = task->event_chunk->size; + flb_task_get_route_metrics(task, ins, &effective_records, &effective_bytes); /* If we are in synchronous mode, flush the next waiting task */ if (ins->flags & FLB_OUTPUT_SYNCHRONOUS) { @@ -351,19 +356,19 @@ static inline int handle_output_event(uint64_t ts, /* A task has finished, delete it */ if (ret == FLB_OK) { /* cmetrics */ - cmt_counter_add(ins->cmt_proc_records, ts, task->event_chunk->total_events, + cmt_counter_add(ins->cmt_proc_records, ts, effective_records, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_proc_bytes, ts, task->event_chunk->size, + cmt_counter_add(ins->cmt_proc_bytes, ts, effective_bytes, 1, (char *[]) {out_name}); if (config->router && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_records_total, ts, - task->event_chunk->total_events, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -378,9 +383,9 @@ static inline int handle_output_event(uint64_t ts, #ifdef FLB_HAVE_METRICS if (ins->metrics) { flb_metrics_sum(FLB_METRIC_OUT_OK_RECORDS, - task->event_chunk->total_events, ins->metrics); + effective_records, ins->metrics); flb_metrics_sum(FLB_METRIC_OUT_OK_BYTES, - task->event_chunk->size, ins->metrics); + effective_bytes, ins->metrics); } #endif /* Inform the user if a 'retry' succedeed */ @@ -416,17 +421,17 @@ static inline int handle_output_event(uint64_t ts, handle_dlq_if_available(config, task, ins, 0); /* cmetrics: output_dropped_records_total */ - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -436,7 +441,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API */ #ifdef FLB_HAVE_METRICS - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif flb_info("[engine] chunk '%s' is not retried (no retry config): " "task_id=%i, input=%s > output=%s (out_id=%i)", @@ -465,17 +470,17 @@ static inline int handle_output_event(uint64_t ts, /* cmetrics */ cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -486,7 +491,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_RETRY_FAILED, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif /* Notify about this failed retry */ flb_error("[engine] chunk '%s' cannot be retried: " @@ -538,7 +543,7 @@ static inline int handle_output_event(uint64_t ts, /* cmetrics */ cmt_counter_inc(ins->cmt_retries, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_retried_records, ts, task->records, + cmt_counter_add(ins->cmt_retried_records, ts, effective_records, 1, (char *[]) {out_name}); cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, @@ -548,7 +553,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD metrics API: update the metrics since a new retry is coming */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_RETRY, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_RETRIED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_RETRIED_RECORDS, effective_records, ins->metrics); #endif } } @@ -556,17 +561,17 @@ static inline int handle_output_event(uint64_t ts, handle_dlq_if_available(config, task, ins, 0); /* cmetrics */ cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {out_name}); - cmt_counter_add(ins->cmt_dropped_records, ts, task->records, + cmt_counter_add(ins->cmt_dropped_records, ts, effective_records, 1, (char *[]) {out_name}); if (config->router && task->event_chunk && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { cmt_counter_add(config->router->logs_drop_records_total, ts, - task->records, + effective_records, 2, (char *[]) {in_name, out_name}); cmt_counter_add(config->router->logs_drop_bytes_total, ts, - task->event_chunk->size, + effective_bytes, 2, (char *[]) {in_name, out_name}); } @@ -577,7 +582,7 @@ static inline int handle_output_event(uint64_t ts, /* OLD API */ #ifdef FLB_HAVE_METRICS flb_metrics_sum(FLB_METRIC_OUT_ERROR, 1, ins->metrics); - flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, task->records, ins->metrics); + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, effective_records, ins->metrics); #endif flb_task_retry_clean(task, ins); @@ -654,6 +659,16 @@ static inline int flb_engine_manager(flb_pipefd_t fd, struct flb_config *config) /* Flush all remaining data */ if (type == 1) { /* Engine type */ if (key == FLB_ENGINE_STOP) { + /* + * Re-entering the STOP handler in flb_engine_start() would reset + * config->event_shutdown.status while the shutdown timerfd is + * still registered, so the dispatcher drops the timer and the + * pipeline thread busy-loops on epoll. + */ + if (config->is_shutting_down) { + flb_debug("[engine] duplicate STOP ignored"); + return 0; + } flb_trace("[engine] flush enqueued data"); flb_engine_flush(config, NULL); return FLB_ENGINE_STOP; diff --git a/src/flb_env.c b/src/flb_env.c index e4cfcff6924..e0b116455b2 100644 --- a/src/flb_env.c +++ b/src/flb_env.c @@ -201,6 +201,9 @@ const char *flb_env_get(struct flb_env *env, const char *key) /* * Given a 'value', lookup for variables, if found, return a new composed * sds string. + * + * Supports bash-style default substitution inside ${...}: + * ${name:-word} — if unset or empty, expand word; do not assign. */ flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) { @@ -211,6 +214,9 @@ flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) int pre_var; int have_var = FLB_FALSE; const char *env_var = NULL; + char *v_sep; + const char *def_val; + int def_len; char *v_start = NULL; char *v_end = NULL; char tmp[4096]; @@ -248,6 +254,18 @@ flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) strncpy(tmp, v_start, v_len); tmp[v_len] = '\0'; have_var = FLB_TRUE; + + /* Bash-style default value expansion :- */ + v_sep = strstr(tmp, ":"); + def_val = NULL; + def_len = 0; + + if (v_sep && (v_sep[1] == '-')) { + def_val = v_sep + 2; + def_len = strlen(def_val); + *v_sep = '\0'; + } + /* Append pre-variable content */ pre_var = (v_start - 2) - (value + i); @@ -264,7 +282,8 @@ flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) /* Lookup the variable in our env-hash */ env_var = flb_env_get(env, tmp); - if (env_var) { + /* Skip env if it's empty and have a fallback defined */ + if (env_var && !(def_val && strlen(env_var) == 0)) { e_len = strlen(env_var); s = buf_append(buf, env_var, e_len); if (!s) { @@ -275,6 +294,16 @@ flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value) buf = s; } } + else if (def_val) { + if (def_len > 0) { + s = buf_append(buf, def_val, def_len); + if (!s) { + flb_sds_destroy(buf); + return NULL; + } + buf = s; + } + } else if (env->warn_unused == FLB_TRUE) { flb_warn("[env] variable ${%s} is used but not set", tmp); } diff --git a/src/flb_filter.c b/src/flb_filter.c index e9298dcfc82..b4bae5b41f8 100644 --- a/src/flb_filter.c +++ b/src/flb_filter.c @@ -331,6 +331,9 @@ int flb_filter_set_property(struct flb_filter_instance *ins, struct flb_kv *kv; len = strlen(k); + if (!v) { + return -1; + } tmp = flb_env_var_translate(ins->config->env, v); if (!tmp) { return -1; diff --git a/src/flb_input_thread.c b/src/flb_input_thread.c index 8604a407081..fe35993cce2 100644 --- a/src/flb_input_thread.c +++ b/src/flb_input_thread.c @@ -59,9 +59,9 @@ static inline int handle_input_event(flb_pipefd_t fd, struct flb_input_instance uint64_t val; struct flb_config *config = ins->config; - bytes = read(fd, &val, sizeof(val)); + bytes = flb_pipe_r(fd, &val, sizeof(val)); if (bytes == -1) { - flb_errno(); + flb_pipe_error(); return -1; } diff --git a/src/flb_mp.c b/src/flb_mp.c index 641616372cd..4cfaaf131bf 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -47,27 +47,29 @@ int flb_mp_count(const void *data, size_t bytes) int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes) { - size_t remaining; + size_t remaining = 0; int count = 0; mpack_reader_t reader; - mpack_reader_init_data(&reader, (const char *) data, bytes); - for (;;) { - remaining = mpack_reader_remaining(&reader, NULL); - if (!remaining) { - break; - } - mpack_discard(&reader); - if (mpack_reader_error(&reader)) { - break; + if (data) { + mpack_reader_init_data(&reader, (const char *) data, bytes); + for (;;) { + remaining = mpack_reader_remaining(&reader, NULL); + if (!remaining) { + break; + } + mpack_discard(&reader); + if (mpack_reader_error(&reader)) { + break; + } + count++; } - count++; + mpack_reader_destroy(&reader); } if (remaining_bytes) { *remaining_bytes = remaining; } - mpack_reader_destroy(&reader); return count; } diff --git a/src/flb_network.c b/src/flb_network.c index 5783d93c50d..92aa601341c 100644 --- a/src/flb_network.c +++ b/src/flb_network.c @@ -1648,18 +1648,38 @@ flb_sockfd_t flb_net_server(const char *port, const char *listen_addr, { flb_sockfd_t fd = -1; int ret; + size_t listen_addr_len; struct addrinfo hints; struct addrinfo *res, *rp; + const char *normalized_listen_addr; + char *listen_addr_copy; + + listen_addr_copy = NULL; + normalized_listen_addr = listen_addr; + + if (listen_addr != NULL && listen_addr[0] == '[') { + listen_addr_len = strlen(listen_addr); + + if (listen_addr_len >= 3 && listen_addr[listen_addr_len - 1] == ']') { + listen_addr_copy = flb_strndup(listen_addr + 1, listen_addr_len - 2); + if (listen_addr_copy != NULL) { + normalized_listen_addr = listen_addr_copy; + } + } + } memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; - ret = getaddrinfo(listen_addr, port, &hints, &res); + ret = getaddrinfo(normalized_listen_addr, port, &hints, &res); if (ret != 0) { flb_warn("net_server: getaddrinfo(listen='%s:%s'): %s", listen_addr, port, gai_strerror(ret)); + if (listen_addr_copy != NULL) { + flb_free(listen_addr_copy); + } return -1; } @@ -1687,6 +1707,10 @@ flb_sockfd_t flb_net_server(const char *port, const char *listen_addr, } freeaddrinfo(res); + if (listen_addr_copy != NULL) { + flb_free(listen_addr_copy); + } + if (rp == NULL) { return -1; } @@ -1698,18 +1722,38 @@ flb_sockfd_t flb_net_server_udp(const char *port, const char *listen_addr, int s { flb_sockfd_t fd = -1; int ret; + size_t listen_addr_len; struct addrinfo hints; struct addrinfo *res, *rp; + const char *normalized_listen_addr; + char *listen_addr_copy; + + listen_addr_copy = NULL; + normalized_listen_addr = listen_addr; + + if (listen_addr != NULL && listen_addr[0] == '[') { + listen_addr_len = strlen(listen_addr); + + if (listen_addr_len >= 3 && listen_addr[listen_addr_len - 1] == ']') { + listen_addr_copy = flb_strndup(listen_addr + 1, listen_addr_len - 2); + if (listen_addr_copy != NULL) { + normalized_listen_addr = listen_addr_copy; + } + } + } memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = AI_PASSIVE; - ret = getaddrinfo(listen_addr, port, &hints, &res); + ret = getaddrinfo(normalized_listen_addr, port, &hints, &res); if (ret != 0) { flb_warn("net_server_udp: getaddrinfo(listen='%s:%s'): %s", listen_addr, port, gai_strerror(ret)); + if (listen_addr_copy != NULL) { + flb_free(listen_addr_copy); + } return -1; } @@ -1734,6 +1778,10 @@ flb_sockfd_t flb_net_server_udp(const char *port, const char *listen_addr, int s } freeaddrinfo(res); + if (listen_addr_copy != NULL) { + flb_free(listen_addr_copy); + } + if (rp == NULL) { return -1; } @@ -2344,4 +2392,4 @@ uint64_t flb_net_htonll(uint64_t value) #else return value; #endif -} \ No newline at end of file +} diff --git a/src/flb_oauth2.c b/src/flb_oauth2.c index 444901f6a81..6c992dd66e6 100644 --- a/src/flb_oauth2.c +++ b/src/flb_oauth2.c @@ -25,6 +25,7 @@ #include #include #include +#include #define free_temporary_buffers() \ if (prot) { \ @@ -54,14 +55,21 @@ int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, { int i; int ret; + int copy_len; int key_len; int val_len; + char *end; int tokens_size = 32; const char *key; const char *val; + unsigned long long parsed_expires_in; jsmn_parser parser; + flb_sds_t new_access_token = NULL; + flb_sds_t new_token_type = NULL; jsmntok_t *t; jsmntok_t *tokens; + char tmp_num[32]; + uint64_t new_expires_in = 0; jsmn_init(&parser); tokens = flb_calloc(1, sizeof(jsmntok_t) * tokens_size); @@ -72,14 +80,14 @@ int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, ret = jsmn_parse(&parser, json_data, json_size, tokens, tokens_size); if (ret <= 0) { - flb_error("[oauth2] cannot parse payload:\n%s", json_data); + flb_error("[oauth2] cannot parse payload (size=%zu)", json_size); flb_free(tokens); return -1; } t = &tokens[0]; if (t->type != JSMN_OBJECT) { - flb_error("[oauth2] invalid JSON response:\n%s", json_data); + flb_error("[oauth2] invalid JSON response (size=%zu)", json_size); flb_free(tokens); return -1; } @@ -100,6 +108,10 @@ int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, key = json_data + t->start; key_len = (t->end - t->start); + if (i + 1 >= ret) { + break; + } + /* Value */ i++; t = &tokens[i]; @@ -107,32 +119,69 @@ int flb_oauth2_parse_json_response(const char *json_data, size_t json_size, val_len = (t->end - t->start); if (key_cmp(key, key_len, "access_token") == 0) { - ctx->access_token = flb_sds_create_len(val, val_len); + if (new_access_token) { + flb_sds_destroy(new_access_token); + } + new_access_token = flb_sds_create_len(val, val_len); + if (!new_access_token) { + flb_errno(); + break; + } } else if (key_cmp(key, key_len, "token_type") == 0) { - ctx->token_type = flb_sds_create_len(val, val_len); + if (new_token_type) { + flb_sds_destroy(new_token_type); + } + new_token_type = flb_sds_create_len(val, val_len); + if (!new_token_type) { + flb_errno(); + break; + } } else if (key_cmp(key, key_len, "expires_in") == 0) { - ctx->expires_in = atol(val); - - /* - * Our internal expiration time must be lower that the one set - * by the remote end-point, so we can use valid cached values - * if a token renewal is in place. So we decrease the expire - * interval -10%. - */ - ctx->expires_in -= (ctx->expires_in * 0.10); + if (val_len <= 0 || val_len >= sizeof(tmp_num)) { + break; + } + + copy_len = val_len < (sizeof(tmp_num) - 1) ? val_len : (sizeof(tmp_num) - 1); + strncpy(tmp_num, val, copy_len); + tmp_num[copy_len] = '\0'; + + if (tmp_num[0] == '-') { + break; + } + + errno = 0; + parsed_expires_in = strtoull(tmp_num, &end, 10); + + if (errno != 0 || end == tmp_num || *end != '\0') { + break; + } + + new_expires_in = parsed_expires_in; + new_expires_in -= (new_expires_in / 10); } } flb_free(tokens); - if (!ctx->access_token || !ctx->token_type || ctx->expires_in < 60) { + + if (!new_access_token || !new_token_type || new_expires_in <= 60) { + flb_sds_destroy(new_access_token); + flb_sds_destroy(new_token_type); + return -1; + } + + if (ctx->access_token) { flb_sds_destroy(ctx->access_token); + } + if (ctx->token_type) { flb_sds_destroy(ctx->token_type); - ctx->expires_in = 0; - return -1; } + ctx->access_token = new_access_token; + ctx->token_type = new_token_type; + ctx->expires_in = new_expires_in; + return 0; } diff --git a/src/flb_parser.c b/src/flb_parser.c index f13243f580f..c1ec6b3e6a0 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -42,8 +42,659 @@ #include #include #include +#include +#include +#include #include +#ifdef FLB_SYSTEM_WINDOWS +struct windows_time_zone { + DYNAMIC_TIME_ZONE_INFORMATION dtzi; +}; + +static int utf8_to_wide(const char *str, wchar_t *buf, int buf_size) +{ + int ret; + + ret = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, buf_size); + if (ret == 0) { + return -1; + } + + return 0; +} + +static int windows_time_zone_lookup(const char *windows_zone, + DYNAMIC_TIME_ZONE_INFORMATION *dtzi) +{ + DWORD index; + DWORD ret; + wchar_t wide_zone[128]; + + if (windows_zone == NULL || dtzi == NULL) { + return -1; + } + + if (utf8_to_wide(windows_zone, wide_zone, + sizeof(wide_zone) / sizeof(wide_zone[0])) != 0) { + return -1; + } + + for (index = 0; ; index++) { + memset(dtzi, 0, sizeof(DYNAMIC_TIME_ZONE_INFORMATION)); + ret = EnumDynamicTimeZoneInformation(index, dtzi); + if (ret == ERROR_NO_MORE_ITEMS) { + break; + } + if (ret != ERROR_SUCCESS) { + continue; + } + if (wcscmp(dtzi->TimeZoneKeyName, wide_zone) == 0) { + return 0; + } + } + + return -1; +} + +static int windows_systemtime_from_tm(const struct tm *tm, SYSTEMTIME *st) +{ + int year; + + year = tm->tm_year + 1900; + if (year < 1601 || year > 30827) { + return -1; + } + + memset(st, 0, sizeof(SYSTEMTIME)); + st->wYear = (WORD) year; + st->wMonth = (WORD) (tm->tm_mon + 1); + st->wDay = (WORD) tm->tm_mday; + st->wHour = (WORD) tm->tm_hour; + st->wMinute = (WORD) tm->tm_min; + st->wSecond = (WORD) tm->tm_sec; + + return 0; +} + +static int windows_time_zone_load(const char *iana_zone, struct windows_time_zone *tz) +{ + int ret; + const char *windows_zone; + + windows_zone = flb_time_iana_zone_to_windows(iana_zone); + if (windows_zone == NULL) { + return -1; + } + + ret = windows_time_zone_lookup(windows_zone, &tz->dtzi); + if (ret != 0) { + return -1; + } + + return 0; +} + +static time_t windows_tm2time_zone(const struct flb_tm *src, struct windows_time_zone *tz) +{ + int ret; + struct tm utc_tm; + SYSTEMTIME local_st; + SYSTEMTIME utc_st; + TIME_ZONE_INFORMATION tzi; + + if (tz == NULL) { + return (time_t) -1; + } + + ret = GetTimeZoneInformationForYear(src->tm.tm_year + 1900, &tz->dtzi, &tzi); + if (ret == 0) { + return (time_t) -1; + } + + ret = windows_systemtime_from_tm(&src->tm, &local_st); + if (ret != 0) { + return (time_t) -1; + } + + ret = TzSpecificLocalTimeToSystemTime(&tzi, &local_st, &utc_st); + if (ret == 0) { + return (time_t) -1; + } + + memset(&utc_tm, 0, sizeof(struct tm)); + utc_tm.tm_year = utc_st.wYear - 1900; + utc_tm.tm_mon = utc_st.wMonth - 1; + utc_tm.tm_mday = utc_st.wDay; + utc_tm.tm_hour = utc_st.wHour; + utc_tm.tm_min = utc_st.wMinute; + utc_tm.tm_sec = utc_st.wSecond; + utc_tm.tm_isdst = 0; + + return timegm(&utc_tm); +} + +static int windows_time2tm_zone(time_t time, struct windows_time_zone *tz, + struct tm *out_tm) +{ + int ret; + struct tm utc_tm; + SYSTEMTIME utc_st; + SYSTEMTIME local_st; + TIME_ZONE_INFORMATION tzi; + + if (tz == NULL) { + return -1; + } + + gmtime_r(&time, &utc_tm); + + ret = GetTimeZoneInformationForYear(utc_tm.tm_year + 1900, &tz->dtzi, &tzi); + if (ret == 0) { + return -1; + } + + if (windows_systemtime_from_tm(&utc_tm, &utc_st) != 0) { + return -1; + } + + ret = SystemTimeToTzSpecificLocalTime(&tzi, &utc_st, &local_st); + if (ret == 0) { + return -1; + } + + memset(out_tm, 0, sizeof(struct tm)); + out_tm->tm_year = local_st.wYear - 1900; + out_tm->tm_mon = local_st.wMonth - 1; + out_tm->tm_mday = local_st.wDay; + out_tm->tm_hour = local_st.wHour; + out_tm->tm_min = local_st.wMinute; + out_tm->tm_sec = local_st.wSecond; + + return 0; +} +#endif + +#ifndef FLB_SYSTEM_WINDOWS +struct tzif_type { + int32_t gmtoff; + unsigned char isdst; +}; + +struct tzif { + int timecnt; + int typecnt; + int default_type; + int64_t *transitions; + unsigned char *transition_types; + struct tzif_type *types; +}; + +static int zoneinfo_file_exists(const char *iana_zone) +{ + int ret; + size_t len; + char path[PATH_MAX]; + const char *tzdir; + struct stat st; + + tzdir = getenv("TZDIR"); + if (tzdir == NULL || tzdir[0] == '\0') { + tzdir = "/usr/share/zoneinfo"; + } + + ret = snprintf(path, sizeof(path), "%s/%s", tzdir, iana_zone); + if (ret < 0) { + return FLB_FALSE; + } + + len = (size_t) ret; + if (len >= sizeof(path)) { + return FLB_FALSE; + } + + if (stat(path, &st) != 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static int zoneinfo_path(const char *iana_zone, char *path, size_t path_size) +{ + int ret; + size_t len; + const char *tzdir; + + tzdir = getenv("TZDIR"); + if (tzdir == NULL || tzdir[0] == '\0') { + tzdir = "/usr/share/zoneinfo"; + } + + ret = snprintf(path, path_size, "%s/%s", tzdir, iana_zone); + if (ret < 0) { + return -1; + } + + len = (size_t) ret; + if (len >= path_size) { + return -1; + } + + return 0; +} + +static uint32_t read_be32(const unsigned char *buf) +{ + return ((uint32_t) buf[0] << 24) | + ((uint32_t) buf[1] << 16) | + ((uint32_t) buf[2] << 8) | + (uint32_t) buf[3]; +} + +static int32_t read_be32s(const unsigned char *buf) +{ + return (int32_t) read_be32(buf); +} + +static int64_t read_be64s(const unsigned char *buf) +{ + uint64_t value; + + value = ((uint64_t) buf[0] << 56) | + ((uint64_t) buf[1] << 48) | + ((uint64_t) buf[2] << 40) | + ((uint64_t) buf[3] << 32) | + ((uint64_t) buf[4] << 24) | + ((uint64_t) buf[5] << 16) | + ((uint64_t) buf[6] << 8) | + (uint64_t) buf[7]; + + return (int64_t) value; +} + +static void tzif_destroy(struct tzif *tz) +{ + if (tz == NULL) { + return; + } + + flb_free(tz->transitions); + flb_free(tz->transition_types); + flb_free(tz->types); +} + +static int tzif_data_size(const unsigned char *header, int time_size, + size_t *out_size) +{ + uint32_t isutcnt; + uint32_t isstdcnt; + uint32_t leapcnt; + uint32_t timecnt; + uint32_t typecnt; + uint32_t charcnt; + size_t size; + + isutcnt = read_be32(header + 20); + isstdcnt = read_be32(header + 24); + leapcnt = read_be32(header + 28); + timecnt = read_be32(header + 32); + typecnt = read_be32(header + 36); + charcnt = read_be32(header + 40); + + size = ((size_t) timecnt * (size_t) time_size) + + (size_t) timecnt + + ((size_t) typecnt * 6) + + (size_t) charcnt + + ((size_t) leapcnt * ((size_t) time_size + 4)) + + (size_t) isstdcnt + + (size_t) isutcnt; + + *out_size = size; + return 0; +} + +static int tzif_parse_data(const unsigned char *buf, size_t size, + int time_size, struct tzif *tz) +{ + int i; + int type; + size_t off; + uint32_t timecnt; + uint32_t typecnt; + + if (size < 44) { + return -1; + } + + timecnt = read_be32(buf + 32); + typecnt = read_be32(buf + 36); + if (typecnt == 0 || timecnt > INT_MAX || typecnt > INT_MAX) { + return -1; + } + + off = 44; + if (off + ((size_t) timecnt * (size_t) time_size) > size) { + return -1; + } + + memset(tz, 0, sizeof(struct tzif)); + tz->timecnt = (int) timecnt; + tz->typecnt = (int) typecnt; + tz->default_type = 0; + + if (timecnt > 0) { + tz->transitions = flb_calloc(timecnt, sizeof(int64_t)); + if (tz->transitions == NULL) { + return -1; + } + } + + for (i = 0; i < (int) timecnt; i++) { + if (time_size == 8) { + tz->transitions[i] = read_be64s(buf + off); + } + else { + tz->transitions[i] = read_be32s(buf + off); + } + off += time_size; + } + + if (off + timecnt > size) { + tzif_destroy(tz); + return -1; + } + + if (timecnt > 0) { + tz->transition_types = flb_malloc(timecnt); + if (tz->transition_types == NULL) { + tzif_destroy(tz); + return -1; + } + memcpy(tz->transition_types, buf + off, timecnt); + } + off += timecnt; + + if (off + ((size_t) typecnt * 6) > size) { + tzif_destroy(tz); + return -1; + } + + tz->types = flb_calloc(typecnt, sizeof(struct tzif_type)); + if (tz->types == NULL) { + tzif_destroy(tz); + return -1; + } + + for (i = 0; i < (int) typecnt; i++) { + tz->types[i].gmtoff = read_be32s(buf + off); + tz->types[i].isdst = buf[off + 4]; + off += 6; + } + + for (i = 0; i < (int) timecnt; i++) { + type = tz->transition_types[i]; + if (type < 0 || type >= (int) typecnt) { + tzif_destroy(tz); + return -1; + } + } + + for (i = 0; i < (int) typecnt; i++) { + if (tz->types[i].isdst == 0) { + tz->default_type = i; + break; + } + } + + return 0; +} + +static int tzif_load(const char *iana_zone, struct tzif *tz) +{ + int ret; + char path[PATH_MAX]; + FILE *fp; + long file_size; + size_t read_size; + size_t block_size; + unsigned char *buf; + const unsigned char *header; + const unsigned char *parse_header; + unsigned char version; + + if (zoneinfo_path(iana_zone, path, sizeof(path)) != 0) { + return -1; + } + + fp = fopen(path, "rb"); + if (fp == NULL) { + return -1; + } + + if (fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return -1; + } + + file_size = ftell(fp); + if (file_size <= 0) { + fclose(fp); + return -1; + } + + rewind(fp); + + buf = flb_malloc((size_t) file_size); + if (buf == NULL) { + fclose(fp); + return -1; + } + + read_size = fread(buf, 1, (size_t) file_size, fp); + fclose(fp); + if (read_size != (size_t) file_size) { + flb_free(buf); + return -1; + } + + if ((size_t) file_size < 44 || memcmp(buf, "TZif", 4) != 0) { + flb_free(buf); + return -1; + } + + header = buf; + version = header[4]; + parse_header = header; + + if (version == '2' || version == '3' || version == '4') { + ret = tzif_data_size(header, 4, &block_size); + if (ret != 0 || 44 + block_size + 44 > (size_t) file_size) { + flb_free(buf); + return -1; + } + + parse_header = buf + 44 + block_size; + if (memcmp(parse_header, "TZif", 4) != 0) { + flb_free(buf); + return -1; + } + + ret = tzif_parse_data(parse_header, + (size_t) file_size - (size_t) (parse_header - buf), + 8, tz); + } + else { + ret = tzif_parse_data(parse_header, (size_t) file_size, 4, tz); + } + + flb_free(buf); + return ret; +} + +static int tzif_type_at_utc(struct tzif *tz, int64_t utc) +{ + int lo; + int hi; + int mid; + + if (tz->timecnt == 0 || utc < tz->transitions[0]) { + return tz->default_type; + } + + lo = 0; + hi = tz->timecnt - 1; + while (lo <= hi) { + mid = lo + ((hi - lo) / 2); + if (tz->transitions[mid] <= utc) { + lo = mid + 1; + } + else { + hi = mid - 1; + } + } + + return tz->transition_types[hi]; +} + +static time_t tzif_tm2time(struct tzif *tz, const struct flb_tm *src) +{ + int i; + int type; + int64_t local_epoch; + int64_t candidate; + struct tm tmp; + + tmp = src->tm; + tmp.tm_isdst = 0; + local_epoch = (int64_t) timegm(&tmp); + + for (i = 0; i < tz->typecnt; i++) { + candidate = local_epoch - (int64_t) tz->types[i].gmtoff; + type = tzif_type_at_utc(tz, candidate); + if (type >= 0 && type < tz->typecnt && + tz->types[type].gmtoff == tz->types[i].gmtoff) { + return (time_t) candidate; + } + } + + type = tzif_type_at_utc(tz, local_epoch); + if (type < 0 || type >= tz->typecnt) { + return (time_t) -1; + } + + return (time_t) (local_epoch - (int64_t) tz->types[type].gmtoff); +} + +static int tzif_time2tm(struct tzif *tz, time_t time, struct tm *out_tm) +{ + int type; + time_t local_time; + + type = tzif_type_at_utc(tz, (int64_t) time); + if (type < 0 || type >= tz->typecnt) { + return -1; + } + + local_time = time + tz->types[type].gmtoff; + gmtime_r(&local_time, out_tm); + + return 0; +} +#endif + +static int validate_time_zone(const char *iana_zone) +{ +#ifdef FLB_SYSTEM_WINDOWS + DYNAMIC_TIME_ZONE_INFORMATION dtzi; +#endif + const char *windows_zone; + + if (iana_zone == NULL || iana_zone[0] == '\0') { + return 0; + } + + /* + * Validate against Fluent Bit's built-in IANA timezone index first. On + * Windows the same entry also gives us the native timezone key. + */ + windows_zone = flb_time_iana_zone_to_windows(iana_zone); + if (windows_zone == NULL) { + return -1; + } + +#ifdef FLB_SYSTEM_WINDOWS + /* Ensure the mapped native timezone is available on this Windows host. */ + if (windows_time_zone_lookup(windows_zone, &dtzi) != 0) { + return -1; + } +#else + if (zoneinfo_file_exists(iana_zone) == FLB_FALSE) { + return -1; + } +#endif + + return 0; +} + +static void *time_zone_data_create(const char *iana_zone) +{ +#ifdef FLB_SYSTEM_WINDOWS + struct windows_time_zone *tz; + + tz = flb_calloc(1, sizeof(struct windows_time_zone)); + if (tz == NULL) { + return NULL; + } + + if (windows_time_zone_load(iana_zone, tz) != 0) { + flb_free(tz); + return NULL; + } + + return tz; +#else + struct tzif *tz; + + tz = flb_calloc(1, sizeof(struct tzif)); + if (tz == NULL) { + return NULL; + } + + if (tzif_load(iana_zone, tz) != 0) { + flb_free(tz); + return NULL; + } + + return tz; +#endif +} + +static void time_zone_data_destroy(void *data) +{ + if (data == NULL) { + return; + } + +#ifndef FLB_SYSTEM_WINDOWS + tzif_destroy((struct tzif *) data); +#endif + flb_free(data); +} + +time_t flb_parser_tm2time_parser(const struct flb_tm *src, struct flb_parser *parser) +{ + if (parser->time_zone && parser->time_with_tz == FLB_FALSE) { +#ifdef FLB_SYSTEM_WINDOWS + return windows_tm2time_zone(src, parser->time_zone_data); +#else + return tzif_tm2time(parser->time_zone_data, src); +#endif + } + + return flb_parser_tm2time(src, parser->time_system_timezone); +} + static inline uint32_t digits10(uint64_t v) { if (v < 10) return 1; if (v < 100) return 2; @@ -140,19 +791,28 @@ static void flb_interim_parser_destroy(struct flb_parser *parser) if (parser->time_key) { flb_free(parser->time_key); } + if (parser->time_zone) { + flb_free(parser->time_zone); + } + if (parser->time_zone_data) { + time_zone_data_destroy(parser->time_zone_data); + } mk_list_del(&parser->_head); flb_free(parser); } -struct flb_parser *flb_parser_create(const char *name, const char *format, +struct flb_parser *flb_parser_create_with_time_zone(const char *name, + const char *format, const char *p_regex, int skip_empty, - const char *time_fmt, const char *time_key, + const char *time_fmt, + const char *time_key, const char *time_offset, int time_keep, int time_strict, int time_system_timezone, + const char *time_zone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, @@ -231,6 +891,12 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, p->name = flb_strdup(name); + if (time_zone && time_zone[0] && !time_fmt) { + flb_error("[parser:%s] time_zone requires time_format", name); + flb_interim_parser_destroy(p); + return NULL; + } + if (time_fmt) { p->time_fmt_full = flb_strdup(time_fmt); if (!p->time_fmt_full) { @@ -319,11 +985,46 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, */ p->time_system_timezone = time_system_timezone; + if (time_zone && time_zone[0]) { + if (time_system_timezone) { + flb_error("[parser:%s] time_zone cannot be combined with " + "time_system_timezone", + name); + flb_interim_parser_destroy(p); + return NULL; + } + if (time_offset && time_offset[0]) { + flb_error("[parser:%s] time_zone cannot be combined with " + "time_offset", + name); + flb_interim_parser_destroy(p); + return NULL; + } + if (validate_time_zone(time_zone) != 0) { + flb_error("[parser:%s] invalid or unavailable time_zone '%s'", + name, time_zone); + flb_interim_parser_destroy(p); + return NULL; + } + p->time_zone = flb_strdup(time_zone); + if (!p->time_zone) { + flb_interim_parser_destroy(p); + return NULL; + } + p->time_zone_data = time_zone_data_create(time_zone); + if (p->time_zone_data == NULL) { + flb_error("[parser:%s] could not load time_zone '%s'", + name, time_zone); + flb_interim_parser_destroy(p); + return NULL; + } + } + /* * Optional fixed timezone offset, only applied if - * not falling back to system timezone. + * not falling back to system timezone or an IANA time_zone. */ - if (!p->time_system_timezone && time_offset) { + if (!p->time_system_timezone && !p->time_zone && time_offset) { diff = 0; len = strlen(time_offset); ret = flb_parser_tzone_offset(time_offset, len, &diff); @@ -347,6 +1048,28 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, return p; } +struct flb_parser *flb_parser_create(const char *name, const char *format, + const char *p_regex, + int skip_empty, + const char *time_fmt, const char *time_key, + const char *time_offset, + int time_keep, + int time_strict, + int time_system_timezone, + int logfmt_no_bare_keys, + struct flb_parser_types *types, + int types_len, + struct mk_list *decoders, + struct flb_config *config) +{ + return flb_parser_create_with_time_zone(name, format, p_regex, skip_empty, + time_fmt, time_key, time_offset, + time_keep, time_strict, + time_system_timezone, NULL, + logfmt_no_bare_keys, types, + types_len, decoders, config); +} + void flb_parser_destroy(struct flb_parser *parser) { int i = 0; @@ -367,6 +1090,12 @@ void flb_parser_destroy(struct flb_parser *parser) if (parser->time_key) { flb_free(parser->time_key); } + if (parser->time_zone) { + flb_free(parser->time_zone); + } + if (parser->time_zone_data) { + time_zone_data_destroy(parser->time_zone_data); + } if (parser->types_len != 0) { for (i=0; itypes_len; i++){ flb_free(parser->types[i].key); @@ -492,6 +1221,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, flb_sds_t time_fmt; flb_sds_t time_key; flb_sds_t time_offset; + flb_sds_t time_zone; flb_sds_t types_str; flb_sds_t tmp_str; int skip_empty; @@ -513,6 +1243,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, time_fmt = NULL; time_key = NULL; time_offset = NULL; + time_zone = NULL; types_str = NULL; tmp_str = NULL; @@ -582,6 +1313,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, /* time_offset (UTC offset) */ time_offset = get_parser_key(config, cf, s, "time_offset"); + /* time_zone (IANA name for naive timestamps) */ + time_zone = get_parser_key(config, cf, s, "time_zone"); + /* logfmt_no_bare_keys */ logfmt_no_bare_keys = FLB_FALSE; tmp_str = get_parser_key(config, cf, s, "logfmt_no_bare_keys"); @@ -603,10 +1337,10 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, decoders = flb_parser_decoder_list_create(s); /* Create the parser context */ - if (!flb_parser_create(name, format, regex, skip_empty, + if (!flb_parser_create_with_time_zone(name, format, regex, skip_empty, time_fmt, time_key, time_offset, time_keep, time_strict, - time_system_timezone, logfmt_no_bare_keys, types, types_len, - decoders, config)) { + time_system_timezone, time_zone, logfmt_no_bare_keys, + types, types_len, decoders, config)) { goto fconf_error; } @@ -627,6 +1361,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, if (time_offset) { flb_sds_destroy(time_offset); } + if (time_zone) { + flb_sds_destroy(time_zone); + } if (types_str) { flb_sds_destroy(types_str); } @@ -663,6 +1400,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, if (time_offset) { flb_sds_destroy(time_offset); } + if (time_zone) { + flb_sds_destroy(time_zone); + } if (types_str) { flb_sds_destroy(types_str); } @@ -1199,7 +1939,25 @@ int flb_parser_time_lookup(const char *time_str, size_t tsize, time_now = now; } - gmtime_r(&time_now, &tmy); + if (parser->time_zone && parser->time_with_tz == FLB_FALSE) { +#ifdef FLB_SYSTEM_WINDOWS + ret = windows_time2tm_zone(time_now, parser->time_zone_data, &tmy); + if (ret != 0) { + return -1; + } +#else + ret = tzif_time2tm(parser->time_zone_data, time_now, &tmy); + if (ret != 0) { + return -1; + } +#endif + } + else if (parser->time_system_timezone == FLB_TRUE) { + localtime_r(&time_now, &tmy); + } + else { + gmtime_r(&time_now, &tmy); + } /* Make the timestamp default to today */ tm->tm.tm_mon = tmy.tm_mon; @@ -1270,7 +2028,7 @@ int flb_parser_time_lookup(const char *time_str, size_t tsize, } } - if (parser->time_with_tz == FLB_FALSE) { + if (parser->time_with_tz == FLB_FALSE && !parser->time_zone) { flb_tm_gmtoff(tm) = parser->time_offset; } diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c index 203793a7e9e..fff252085a2 100644 --- a/src/flb_parser_json.c +++ b/src/flb_parser_json.c @@ -211,7 +211,7 @@ int flb_parser_json_do(struct flb_parser *parser, skip = map_size; } else { - time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + time_lookup = flb_parser_tm2time_parser(&tm, parser); } /* Compose a new map without the time_key field */ diff --git a/src/flb_parser_logfmt.c b/src/flb_parser_logfmt.c index ad372a07121..5f2681f2d8c 100644 --- a/src/flb_parser_logfmt.c +++ b/src/flb_parser_logfmt.c @@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + *time_lookup = flb_parser_tm2time_parser(&tm, parser); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_ltsv.c b/src/flb_parser_ltsv.c index ea099d83bf6..7661e10b80f 100644 --- a/src/flb_parser_ltsv.c +++ b/src/flb_parser_ltsv.c @@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + *time_lookup = flb_parser_tm2time_parser(&tm, parser); } time_found = FLB_TRUE; } diff --git a/src/flb_parser_regex.c b/src/flb_parser_regex.c index b1baee2d693..a80e1c25906 100644 --- a/src/flb_parser_regex.c +++ b/src/flb_parser_regex.c @@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value, } pcb->time_frac = frac; - pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + pcb->time_lookup = flb_parser_tm2time_parser(&tm, parser); if (parser->time_keep == FLB_FALSE) { pcb->num_skipped++; diff --git a/src/flb_plugin_proxy.c b/src/flb_plugin_proxy.c index 578066eeb20..b812f8a370b 100644 --- a/src/flb_plugin_proxy.c +++ b/src/flb_plugin_proxy.c @@ -362,6 +362,14 @@ static int flb_proxy_register_output(struct flb_plugin_proxy *proxy, out->flags = def->flags; out->name = flb_strdup(def->name); + /* If event_type is unset (0) then default to logs (this is the current behavior) */ + if (def->event_type == 0) { + out->event_type = FLB_OUTPUT_LOGS; + } + else { + out->event_type = def->event_type; + } + out->description = def->description; mk_list_add(&out->_head, &config->out_plugins); @@ -396,6 +404,7 @@ static int flb_proxy_register_input(struct flb_plugin_proxy *proxy, in->flags = def->flags | FLB_INPUT_THREADED; in->name = flb_strdup(def->name); in->description = def->description; + mk_list_add(&in->_head, &config->in_plugins); /* @@ -612,7 +621,7 @@ struct flb_plugin_proxy *flb_plugin_proxy_create(const char *dso_path, int type, return NULL; } - proxy->def = flb_malloc(sizeof(struct flb_plugin_proxy_def)); + proxy->def = flb_calloc(1, sizeof(struct flb_plugin_proxy_def)); if (!proxy->def) { flb_errno(); dlclose(handle); diff --git a/src/flb_processor.c b/src/flb_processor.c index d3522fd9a6d..4cd199140a9 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -579,9 +580,11 @@ static int flb_processor_unit_set_condition(struct flb_processor_unit *pu, struc int flb_processor_unit_set_property(struct flb_processor_unit *pu, const char *k, struct cfl_variant *v) { - struct cfl_variant *val; int i; int ret; + char buf[64]; + flb_sds_t str_val; + struct cfl_variant *val; /* Handle the "condition" property for processor units */ if (strcasecmp(k, "condition") == 0) { @@ -596,7 +599,33 @@ int flb_processor_unit_set_property(struct flb_processor_unit *pu, const char *k else if (v->type == CFL_VARIANT_ARRAY) { for (i = 0; i < v->data.as_array->entry_count; i++) { val = v->data.as_array->entries[i]; - ret = flb_filter_set_property(pu->ctx, k, val->data.as_string); + + if (val->type == CFL_VARIANT_STRING) { + ret = flb_filter_set_property(pu->ctx, k, val->data.as_string); + } + else if (val->type == CFL_VARIANT_INT) { + snprintf(buf, sizeof(buf), "%" PRId64, val->data.as_int64); + str_val = flb_sds_create(buf); + ret = (str_val != NULL) ? flb_filter_set_property(pu->ctx, k, str_val) : -1; + flb_sds_destroy(str_val); + } + else if (val->type == CFL_VARIANT_UINT) { + snprintf(buf, sizeof(buf), "%" PRIu64, val->data.as_uint64); + str_val = flb_sds_create(buf); + ret = (str_val != NULL) ? flb_filter_set_property(pu->ctx, k, str_val) : -1; + flb_sds_destroy(str_val); + } + else if (val->type == CFL_VARIANT_DOUBLE) { + snprintf(buf, sizeof(buf), "%g", val->data.as_double); + str_val = flb_sds_create(buf); + ret = (str_val != NULL) ? flb_filter_set_property(pu->ctx, k, str_val) : -1; + flb_sds_destroy(str_val); + } + else { + flb_error("[processor] property '%s': array element type %d not supported for filter", + k, val->type); + return -1; + } if (ret == -1) { return ret; diff --git a/src/flb_regex.c b/src/flb_regex.c index 83994d2549e..a6702894046 100644 --- a/src/flb_regex.c +++ b/src/flb_regex.c @@ -129,7 +129,7 @@ static int str_to_regex(const char *pattern, OnigRegex *reg) option = check_option(start, end, &new_end); - if (pattern[0] == '/' && pattern[len - 1] == '/') { + if (len > 1 && pattern[0] == '/' && pattern[len - 1] == '/') { start++; end--; } diff --git a/src/flb_router_config.c b/src/flb_router_config.c index 81e3241cd39..b2c7a9f0e18 100644 --- a/src/flb_router_config.c +++ b/src/flb_router_config.c @@ -1080,9 +1080,36 @@ static int parse_routes_block(struct cfl_variant *variant, return -1; } +static struct flb_input_instance *find_input_instance_by_index(struct flb_config *config, + size_t input_index) +{ + size_t index; + struct mk_list *head; + struct flb_input_instance *ins; + + if (!config) { + return NULL; + } + + index = 0; + + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (index == input_index) { + return ins; + } + + index++; + } + + return NULL; +} + static int parse_input_section(struct flb_cf_section *section, struct cfl_list *input_routes, - struct flb_config *config) + struct flb_config *config, + size_t input_index) { uint32_t mask; size_t before_count; @@ -1136,7 +1163,7 @@ static int parse_input_section(struct flb_cf_section *section, cfl_list_init(&input->processors); cfl_list_init(&input->routes); input->has_alias = FLB_FALSE; - input->instance = NULL; + input->instance = find_input_instance_by_index(config, input_index); input->plugin_name = copy_from_cfl_sds(name_var->data.as_string); if (!input->plugin_name) { @@ -1209,6 +1236,7 @@ int flb_router_config_parse(struct flb_cf *cf, { struct mk_list *head; struct flb_cf_section *section; + size_t input_index; int routes_found = FLB_FALSE; int ret; @@ -1218,9 +1246,10 @@ int flb_router_config_parse(struct flb_cf *cf, cfl_list_init(input_routes); + input_index = 0; mk_list_foreach(head, &cf->inputs) { section = mk_list_entry(head, struct flb_cf_section, _head_section); - ret = parse_input_section(section, input_routes, config); + ret = parse_input_section(section, input_routes, config, input_index); if (ret == -1) { flb_router_routes_destroy(input_routes); cfl_list_init(input_routes); @@ -1229,6 +1258,7 @@ int flb_router_config_parse(struct flb_cf *cf, else if (ret == 1) { routes_found = FLB_TRUE; } + input_index++; } if (cfl_list_is_empty(input_routes) == 1) { diff --git a/src/flb_scheduler.c b/src/flb_scheduler.c index d4fa442e9ae..d966f492b87 100644 --- a/src/flb_scheduler.c +++ b/src/flb_scheduler.c @@ -751,6 +751,9 @@ struct flb_sched *flb_sched_create(struct flb_config *config, sched->config = config; sched->evl = evl; + sched->ch_events[0] = -1; + sched->ch_events[1] = -1; + MK_EVENT_ZERO(&sched->event); /* Initialize lists */ mk_list_init(&sched->requests); @@ -791,7 +794,7 @@ struct flb_sched *flb_sched_create(struct flb_config *config, ret = mk_event_channel_create(sched->evl, &sched->ch_events[0], &sched->ch_events[1], - sched); + &sched->event); if (ret == -1) { flb_sched_destroy(sched); return NULL; @@ -849,6 +852,15 @@ int flb_sched_destroy(struct flb_sched *sched) c++; } + if (sched->ch_events[0] != -1) { + mk_event_channel_destroy(sched->evl, + sched->ch_events[0], + sched->ch_events[1], + &sched->event); + sched->ch_events[0] = -1; + sched->ch_events[1] = -1; + } + flb_free(sched); return c; } diff --git a/src/flb_snappy.c b/src/flb_snappy.c index c4c16b0ade6..74595e8a8f0 100644 --- a/src/flb_snappy.c +++ b/src/flb_snappy.c @@ -170,8 +170,9 @@ int flb_snappy_uncompress_framed_data(char *in_data, size_t in_len, frame_type = *((uint8_t *) &frame_buffer[0]); - frame_length = *((uint32_t *) &frame_buffer[1]); - frame_length &= 0x00FFFFFF; + frame_length = ((uint32_t)((unsigned char) frame_buffer[1])) | + ((uint32_t)((unsigned char) frame_buffer[2]) << 8) | + ((uint32_t)((unsigned char) frame_buffer[3]) << 16); frame_body = &frame_buffer[4]; diff --git a/src/flb_task.c b/src/flb_task.c index 5e671f5ee40..b7d4f24166a 100644 --- a/src/flb_task.c +++ b/src/flb_task.c @@ -708,6 +708,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = task->event_chunk->total_events; + route->bytes = task->event_chunk->size; route->out = stored_matches[stored_match_index]; mk_list_add(&route->_head, &task->routes); direct_count++; @@ -810,6 +812,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = task->event_chunk->total_events; + route->bytes = task->event_chunk->size; route->out = o_ins; mk_list_add(&route->_head, &task->routes); direct_count++; @@ -856,6 +860,8 @@ struct flb_task *flb_task_create(uint64_t ref_id, } route->status = FLB_TASK_ROUTE_INACTIVE; + route->records = task->event_chunk->total_events; + route->bytes = task->event_chunk->size; route->out = o_ins; mk_list_add(&route->_head, &task->routes); count++; diff --git a/src/flb_time.c b/src/flb_time.c index c511d474ff9..6a1c94e0287 100644 --- a/src/flb_time.c +++ b/src/flb_time.c @@ -442,3 +442,5 @@ long flb_time_tz_offset_to_second() return diff; } + +#include "flb_time_tz.c" diff --git a/src/flb_time_tz.c b/src/flb_time_tz.c new file mode 100644 index 00000000000..936b5fae1e5 --- /dev/null +++ b/src/flb_time_tz.c @@ -0,0 +1,649 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +struct flb_time_tz_map { + const char *windows; + const char *iana; + /* Standard UTC offset in seconds; dynamic DST offsets require IANA tzdb. */ + long utc_offset; +}; + +#define FLB_TZ_UTC_OFFSET(hours, minutes) \ + ((((hours) * 60) + (minutes)) * 60L) + +static const struct flb_time_tz_map windows_iana_timezones[] = { + { "Dateline Standard Time", "Etc/GMT+12", FLB_TZ_UTC_OFFSET(-12, 0) }, + { "UTC-11", "Etc/GMT+11", FLB_TZ_UTC_OFFSET(-11, 0) }, + { "UTC-11", "Pacific/Pago_Pago", FLB_TZ_UTC_OFFSET(-11, 0) }, + { "UTC-11", "Pacific/Niue", FLB_TZ_UTC_OFFSET(-11, 0) }, + { "UTC-11", "Pacific/Midway", FLB_TZ_UTC_OFFSET(-11, 0) }, + { "Aleutian Standard Time", "America/Adak", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Hawaiian Standard Time", "Pacific/Honolulu", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Hawaiian Standard Time", "Pacific/Rarotonga", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Hawaiian Standard Time", "Pacific/Tahiti", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Hawaiian Standard Time", "Pacific/Johnston", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Hawaiian Standard Time", "Etc/GMT+10", FLB_TZ_UTC_OFFSET(-10, 0) }, + { "Marquesas Standard Time", "Pacific/Marquesas", FLB_TZ_UTC_OFFSET(-9, -30) }, + { "Alaskan Standard Time", "America/Anchorage", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Alaskan Standard Time", "America/Juneau", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Alaskan Standard Time", "America/Metlakatla", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Alaskan Standard Time", "America/Nome", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Alaskan Standard Time", "America/Sitka", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Alaskan Standard Time", "America/Yakutat", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "UTC-09", "Etc/GMT+9", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "UTC-09", "Pacific/Gambier", FLB_TZ_UTC_OFFSET(-9, 0) }, + { "Pacific Standard Time (Mexico)", "America/Tijuana", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "Pacific Standard Time (Mexico)", "America/Santa_Isabel", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "UTC-08", "Etc/GMT+8", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "UTC-08", "Pacific/Pitcairn", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "Pacific Standard Time", "America/Los_Angeles", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "Pacific Standard Time", "America/Vancouver", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "Pacific Standard Time", "PST8PDT", FLB_TZ_UTC_OFFSET(-8, 0) }, + { "US Mountain Standard Time", "America/Phoenix", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "US Mountain Standard Time", "America/Creston", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "US Mountain Standard Time", "America/Dawson_Creek", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "US Mountain Standard Time", "America/Fort_Nelson", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "US Mountain Standard Time", "America/Hermosillo", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "US Mountain Standard Time", "Etc/GMT+7", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time (Mexico)", "America/Chihuahua", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time (Mexico)", "America/Mazatlan", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Denver", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Edmonton", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Cambridge_Bay", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Inuvik", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Yellowknife", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Ojinaga", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "America/Boise", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Mountain Standard Time", "MST7MDT", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Yukon Standard Time", "America/Whitehorse", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Yukon Standard Time", "America/Dawson", FLB_TZ_UTC_OFFSET(-7, 0) }, + { "Central America Standard Time", "America/Guatemala", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "America/Belize", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "America/Costa_Rica", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "Pacific/Galapagos", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "America/Tegucigalpa", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "America/Managua", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "America/El_Salvador", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central America Standard Time", "Etc/GMT+6", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Chicago", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Winnipeg", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Rainy_River", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Rankin_Inlet", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Resolute", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Matamoros", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Indiana/Knox", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Indiana/Tell_City", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/Menominee", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/North_Dakota/Beulah", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/North_Dakota/Center", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "America/North_Dakota/New_Salem", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time", "CST6CDT", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Easter Island Standard Time", "Pacific/Easter", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time (Mexico)", "America/Mexico_City", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time (Mexico)", "America/Bahia_Banderas", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time (Mexico)", "America/Merida", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Central Standard Time (Mexico)", "America/Monterrey", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Canada Central Standard Time", "America/Regina", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "Canada Central Standard Time", "America/Swift_Current", FLB_TZ_UTC_OFFSET(-6, 0) }, + { "SA Pacific Standard Time", "America/Bogota", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Rio_Branco", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Eirunepe", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Coral_Harbour", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Guayaquil", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Jamaica", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Cayman", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Panama", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "America/Lima", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "SA Pacific Standard Time", "Etc/GMT+5", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time (Mexico)", "America/Cancun", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/New_York", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Nassau", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Toronto", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Iqaluit", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Montreal", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Nipigon", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Pangnirtung", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Thunder_Bay", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Detroit", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Indiana/Petersburg", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Indiana/Vincennes", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Indiana/Winamac", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Kentucky/Monticello", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "America/Louisville", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Eastern Standard Time", "EST5EDT", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Haiti Standard Time", "America/Port-au-Prince", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Cuba Standard Time", "America/Havana", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "US Eastern Standard Time", "America/Indianapolis", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "US Eastern Standard Time", "America/Indiana/Marengo", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "US Eastern Standard Time", "America/Indiana/Vevay", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Turks And Caicos Standard Time", "America/Grand_Turk", FLB_TZ_UTC_OFFSET(-5, 0) }, + { "Paraguay Standard Time", "America/Asuncion", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "America/Halifax", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "Atlantic/Bermuda", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "America/Glace_Bay", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "America/Goose_Bay", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "America/Moncton", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Atlantic Standard Time", "America/Thule", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Venezuela Standard Time", "America/Caracas", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Central Brazilian Standard Time", "America/Cuiaba", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Central Brazilian Standard Time", "America/Campo_Grande", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/La_Paz", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Antigua", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Anguilla", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Aruba", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Barbados", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/St_Barthelemy", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Kralendijk", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Manaus", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Boa_Vista", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Porto_Velho", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Blanc-Sablon", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Curacao", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Dominica", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Santo_Domingo", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Grenada", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Guadeloupe", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Guyana", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/St_Kitts", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/St_Lucia", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Marigot", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Martinique", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Montserrat", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Puerto_Rico", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Lower_Princes", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Port_of_Spain", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/St_Vincent", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/Tortola", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "America/St_Thomas", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "SA Western Standard Time", "Etc/GMT+4", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Pacific SA Standard Time", "America/Santiago", FLB_TZ_UTC_OFFSET(-4, 0) }, + { "Newfoundland Standard Time", "America/St_Johns", FLB_TZ_UTC_OFFSET(-3, -30) }, + { "Tocantins Standard Time", "America/Araguaina", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "E. South America Standard Time", "America/Sao_Paulo", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Cayenne", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "Antarctica/Rothera", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "Antarctica/Palmer", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Fortaleza", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Belem", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Maceio", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Santarem", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Recife", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "Atlantic/Stanley", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "America/Paramaribo", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "SA Eastern Standard Time", "Etc/GMT+3", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Buenos_Aires", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/La_Rioja", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/Rio_Gallegos", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/Salta", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/San_Juan", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/San_Luis", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/Tucuman", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Argentina/Ushuaia", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Catamarca", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Cordoba", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Jujuy", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Argentina Standard Time", "America/Mendoza", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Greenland Standard Time", "America/Nuuk", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Greenland Standard Time", "America/Godthab", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Montevideo Standard Time", "America/Montevideo", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Magallanes Standard Time", "America/Punta_Arenas", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Saint Pierre Standard Time", "America/Miquelon", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "Bahia Standard Time", "America/Bahia", FLB_TZ_UTC_OFFSET(-3, 0) }, + { "UTC-02", "Etc/GMT+2", FLB_TZ_UTC_OFFSET(-2, 0) }, + { "UTC-02", "America/Noronha", FLB_TZ_UTC_OFFSET(-2, 0) }, + { "UTC-02", "Atlantic/South_Georgia", FLB_TZ_UTC_OFFSET(-2, 0) }, + { "Azores Standard Time", "Atlantic/Azores", FLB_TZ_UTC_OFFSET(-1, 0) }, + { "Azores Standard Time", "America/Scoresbysund", FLB_TZ_UTC_OFFSET(-1, 0) }, + { "Cape Verde Standard Time", "Atlantic/Cape_Verde", FLB_TZ_UTC_OFFSET(-1, 0) }, + { "Cape Verde Standard Time", "Etc/GMT+1", FLB_TZ_UTC_OFFSET(-1, 0) }, + { "UTC", "Etc/UTC", FLB_TZ_UTC_OFFSET(0, 0) }, + { "UTC", "Etc/GMT", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/London", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Atlantic/Canary", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Atlantic/Faeroe", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/Guernsey", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/Dublin", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/Isle_of_Man", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/Jersey", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Europe/Lisbon", FLB_TZ_UTC_OFFSET(0, 0) }, + { "GMT Standard Time", "Atlantic/Madeira", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Atlantic/Reykjavik", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Ouagadougou", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Abidjan", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Accra", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "America/Danmarkshavn", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Banjul", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Conakry", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Bissau", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Monrovia", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Bamako", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Nouakchott", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Atlantic/St_Helena", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Freetown", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Dakar", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Greenwich Standard Time", "Africa/Lome", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Sao Tome Standard Time", "Africa/Sao_Tome", FLB_TZ_UTC_OFFSET(0, 0) }, + { "Morocco Standard Time", "Africa/Casablanca", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Morocco Standard Time", "Africa/El_Aaiun", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Berlin", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Andorra", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Vienna", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Zurich", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Busingen", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Gibraltar", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Rome", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Vaduz", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Luxembourg", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Monaco", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Malta", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Amsterdam", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Oslo", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Stockholm", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Arctic/Longyearbyen", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/San_Marino", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Europe Standard Time", "Europe/Vatican", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Budapest", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Tirane", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Prague", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Podgorica", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Belgrade", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Ljubljana", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central Europe Standard Time", "Europe/Bratislava", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Romance Standard Time", "Europe/Paris", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Romance Standard Time", "Europe/Brussels", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Romance Standard Time", "Europe/Copenhagen", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Romance Standard Time", "Africa/Ceuta", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central European Standard Time", "Europe/Warsaw", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central European Standard Time", "Europe/Sarajevo", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central European Standard Time", "Europe/Zagreb", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Central European Standard Time", "Europe/Skopje", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Lagos", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Luanda", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Porto-Novo", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Kinshasa", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Bangui", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Brazzaville", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Douala", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Algiers", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Libreville", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Malabo", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Niamey", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Ndjamena", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Africa/Tunis", FLB_TZ_UTC_OFFSET(1, 0) }, + { "W. Central Africa Standard Time", "Etc/GMT-1", FLB_TZ_UTC_OFFSET(1, 0) }, + { "Jordan Standard Time", "Asia/Amman", FLB_TZ_UTC_OFFSET(2, 0) }, + { "GTB Standard Time", "Europe/Bucharest", FLB_TZ_UTC_OFFSET(2, 0) }, + { "GTB Standard Time", "Asia/Nicosia", FLB_TZ_UTC_OFFSET(2, 0) }, + { "GTB Standard Time", "Asia/Famagusta", FLB_TZ_UTC_OFFSET(2, 0) }, + { "GTB Standard Time", "Europe/Athens", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Middle East Standard Time", "Asia/Beirut", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Egypt Standard Time", "Africa/Cairo", FLB_TZ_UTC_OFFSET(2, 0) }, + { "E. Europe Standard Time", "Europe/Chisinau", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Syria Standard Time", "Asia/Damascus", FLB_TZ_UTC_OFFSET(2, 0) }, + { "West Bank Standard Time", "Asia/Hebron", FLB_TZ_UTC_OFFSET(2, 0) }, + { "West Bank Standard Time", "Asia/Gaza", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Johannesburg", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Bujumbura", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Gaborone", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Lubumbashi", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Maseru", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Blantyre", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Maputo", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Kigali", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Mbabane", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Lusaka", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Africa/Harare", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Africa Standard Time", "Etc/GMT-2", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Kyiv", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Kiev", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Mariehamn", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Sofia", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Tallinn", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Helsinki", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Vilnius", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Riga", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Uzhgorod", FLB_TZ_UTC_OFFSET(2, 0) }, + { "FLE Standard Time", "Europe/Zaporozhye", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Israel Standard Time", "Asia/Jerusalem", FLB_TZ_UTC_OFFSET(2, 0) }, + { "South Sudan Standard Time", "Africa/Juba", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Kaliningrad Standard Time", "Europe/Kaliningrad", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Sudan Standard Time", "Africa/Khartoum", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Libya Standard Time", "Africa/Tripoli", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Namibia Standard Time", "Africa/Windhoek", FLB_TZ_UTC_OFFSET(2, 0) }, + { "Arabic Standard Time", "Asia/Baghdad", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Turkey Standard Time", "Europe/Istanbul", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Arab Standard Time", "Asia/Riyadh", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Arab Standard Time", "Asia/Bahrain", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Arab Standard Time", "Asia/Kuwait", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Arab Standard Time", "Asia/Qatar", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Arab Standard Time", "Asia/Aden", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Belarus Standard Time", "Europe/Minsk", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Russian Standard Time", "Europe/Moscow", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Russian Standard Time", "Europe/Kirov", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Russian Standard Time", "Europe/Simferopol", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Nairobi", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Antarctica/Syowa", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Djibouti", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Asmera", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Addis_Ababa", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Indian/Comoro", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Indian/Antananarivo", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Mogadishu", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Dar_es_Salaam", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Africa/Kampala", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Indian/Mayotte", FLB_TZ_UTC_OFFSET(3, 0) }, + { "E. Africa Standard Time", "Etc/GMT-3", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Iran Standard Time", "Asia/Tehran", FLB_TZ_UTC_OFFSET(3, 30) }, + { "Arabian Standard Time", "Asia/Dubai", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Arabian Standard Time", "Asia/Muscat", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Arabian Standard Time", "Etc/GMT-4", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Astrakhan Standard Time", "Europe/Astrakhan", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Astrakhan Standard Time", "Europe/Ulyanovsk", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Azerbaijan Standard Time", "Asia/Baku", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Russia Time Zone 3", "Europe/Samara", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Mauritius Standard Time", "Indian/Mauritius", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Mauritius Standard Time", "Indian/Reunion", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Mauritius Standard Time", "Indian/Mahe", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Saratov Standard Time", "Europe/Saratov", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Georgian Standard Time", "Asia/Tbilisi", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Volgograd Standard Time", "Europe/Volgograd", FLB_TZ_UTC_OFFSET(3, 0) }, + { "Caucasus Standard Time", "Asia/Yerevan", FLB_TZ_UTC_OFFSET(4, 0) }, + { "Afghanistan Standard Time", "Asia/Kabul", FLB_TZ_UTC_OFFSET(4, 30) }, + { "West Asia Standard Time", "Asia/Tashkent", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Antarctica/Mawson", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Oral", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Aqtau", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Aqtobe", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Atyrau", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Indian/Maldives", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Indian/Kerguelen", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Dushanbe", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Ashgabat", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Asia/Samarkand", FLB_TZ_UTC_OFFSET(5, 0) }, + { "West Asia Standard Time", "Etc/GMT-5", FLB_TZ_UTC_OFFSET(5, 0) }, + { "Ekaterinburg Standard Time", "Asia/Yekaterinburg", FLB_TZ_UTC_OFFSET(5, 0) }, + { "Pakistan Standard Time", "Asia/Karachi", FLB_TZ_UTC_OFFSET(5, 0) }, + { "Qyzylorda Standard Time", "Asia/Qyzylorda", FLB_TZ_UTC_OFFSET(5, 0) }, + { "India Standard Time", "Asia/Kolkata", FLB_TZ_UTC_OFFSET(5, 30) }, + { "India Standard Time", "Asia/Calcutta", FLB_TZ_UTC_OFFSET(5, 30) }, + { "Sri Lanka Standard Time", "Asia/Colombo", FLB_TZ_UTC_OFFSET(5, 30) }, + { "Nepal Standard Time", "Asia/Kathmandu", FLB_TZ_UTC_OFFSET(5, 45) }, + { "Nepal Standard Time", "Asia/Katmandu", FLB_TZ_UTC_OFFSET(5, 45) }, + { "Central Asia Standard Time", "Asia/Almaty", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Antarctica/Vostok", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Asia/Urumqi", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Indian/Chagos", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Asia/Bishkek", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Asia/Qostanay", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Central Asia Standard Time", "Etc/GMT-6", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Bangladesh Standard Time", "Asia/Dhaka", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Bangladesh Standard Time", "Asia/Thimphu", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Omsk Standard Time", "Asia/Omsk", FLB_TZ_UTC_OFFSET(6, 0) }, + { "Myanmar Standard Time", "Asia/Yangon", FLB_TZ_UTC_OFFSET(6, 30) }, + { "Myanmar Standard Time", "Asia/Rangoon", FLB_TZ_UTC_OFFSET(6, 30) }, + { "Myanmar Standard Time", "Indian/Cocos", FLB_TZ_UTC_OFFSET(6, 30) }, + { "SE Asia Standard Time", "Asia/Bangkok", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Antarctica/Davis", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Indian/Christmas", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Jakarta", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Pontianak", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Phnom_Penh", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Vientiane", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Ho_Chi_Minh", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Asia/Saigon", FLB_TZ_UTC_OFFSET(7, 0) }, + { "SE Asia Standard Time", "Etc/GMT-7", FLB_TZ_UTC_OFFSET(7, 0) }, + { "Altai Standard Time", "Asia/Barnaul", FLB_TZ_UTC_OFFSET(7, 0) }, + { "W. Mongolia Standard Time", "Asia/Hovd", FLB_TZ_UTC_OFFSET(7, 0) }, + { "North Asia Standard Time", "Asia/Krasnoyarsk", FLB_TZ_UTC_OFFSET(7, 0) }, + { "North Asia Standard Time", "Asia/Novokuznetsk", FLB_TZ_UTC_OFFSET(7, 0) }, + { "N. Central Asia Standard Time", "Asia/Novosibirsk", FLB_TZ_UTC_OFFSET(7, 0) }, + { "Tomsk Standard Time", "Asia/Tomsk", FLB_TZ_UTC_OFFSET(7, 0) }, + { "China Standard Time", "Asia/Shanghai", FLB_TZ_UTC_OFFSET(8, 0) }, + { "China Standard Time", "Asia/Hong_Kong", FLB_TZ_UTC_OFFSET(8, 0) }, + { "China Standard Time", "Asia/Macau", FLB_TZ_UTC_OFFSET(8, 0) }, + { "North Asia East Standard Time", "Asia/Irkutsk", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Singapore", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Brunei", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Makassar", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Kuching", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Kuala_Lumpur", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Asia/Manila", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Singapore Standard Time", "Etc/GMT-8", FLB_TZ_UTC_OFFSET(8, 0) }, + { "W. Australia Standard Time", "Australia/Perth", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Taipei Standard Time", "Asia/Taipei", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Ulaanbaatar Standard Time", "Asia/Ulaanbaatar", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Ulaanbaatar Standard Time", "Asia/Choibalsan", FLB_TZ_UTC_OFFSET(8, 0) }, + { "Aus Central W. Standard Time", "Australia/Eucla", FLB_TZ_UTC_OFFSET(8, 45) }, + { "Transbaikal Standard Time", "Asia/Chita", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Tokyo Standard Time", "Asia/Tokyo", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Tokyo Standard Time", "Asia/Jayapura", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Tokyo Standard Time", "Pacific/Palau", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Tokyo Standard Time", "Asia/Dili", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Tokyo Standard Time", "Etc/GMT-9", FLB_TZ_UTC_OFFSET(9, 0) }, + { "North Korea Standard Time", "Asia/Pyongyang", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Korea Standard Time", "Asia/Seoul", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Yakutsk Standard Time", "Asia/Yakutsk", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Yakutsk Standard Time", "Asia/Khandyga", FLB_TZ_UTC_OFFSET(9, 0) }, + { "Cen. Australia Standard Time", "Australia/Adelaide", FLB_TZ_UTC_OFFSET(9, 30) }, + { "Cen. Australia Standard Time", "Australia/Broken_Hill", FLB_TZ_UTC_OFFSET(9, 30) }, + { "AUS Central Standard Time", "Australia/Darwin", FLB_TZ_UTC_OFFSET(9, 30) }, + { "E. Australia Standard Time", "Australia/Brisbane", FLB_TZ_UTC_OFFSET(10, 0) }, + { "E. Australia Standard Time", "Australia/Lindeman", FLB_TZ_UTC_OFFSET(10, 0) }, + { "AUS Eastern Standard Time", "Australia/Sydney", FLB_TZ_UTC_OFFSET(10, 0) }, + { "AUS Eastern Standard Time", "Australia/Melbourne", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Pacific/Port_Moresby", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Antarctica/DumontDUrville", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Pacific/Truk", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Pacific/Chuuk", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Pacific/Guam", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Pacific/Saipan", FLB_TZ_UTC_OFFSET(10, 0) }, + { "West Pacific Standard Time", "Etc/GMT-10", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Tasmania Standard Time", "Australia/Hobart", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Tasmania Standard Time", "Australia/Currie", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Tasmania Standard Time", "Antarctica/Macquarie", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Vladivostok Standard Time", "Asia/Vladivostok", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Vladivostok Standard Time", "Asia/Ust-Nera", FLB_TZ_UTC_OFFSET(10, 0) }, + { "Lord Howe Standard Time", "Australia/Lord_Howe", FLB_TZ_UTC_OFFSET(10, 30) }, + { "Bougainville Standard Time", "Pacific/Bougainville", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Russia Time Zone 10", "Asia/Srednekolymsk", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Magadan Standard Time", "Asia/Magadan", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Norfolk Standard Time", "Pacific/Norfolk", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Sakhalin Standard Time", "Asia/Sakhalin", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Guadalcanal", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Antarctica/Casey", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Pohnpei", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Ponape", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Kosrae", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Noumea", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Pacific/Efate", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Central Pacific Standard Time", "Etc/GMT-11", FLB_TZ_UTC_OFFSET(11, 0) }, + { "Russia Time Zone 11", "Asia/Kamchatka", FLB_TZ_UTC_OFFSET(12, 0) }, + { "Russia Time Zone 11", "Asia/Anadyr", FLB_TZ_UTC_OFFSET(12, 0) }, + { "New Zealand Standard Time", "Pacific/Auckland", FLB_TZ_UTC_OFFSET(12, 0) }, + { "New Zealand Standard Time", "Antarctica/McMurdo", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Etc/GMT-12", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Tarawa", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Majuro", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Kwajalein", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Nauru", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Funafuti", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Wake", FLB_TZ_UTC_OFFSET(12, 0) }, + { "UTC+12", "Pacific/Wallis", FLB_TZ_UTC_OFFSET(12, 0) }, + { "Fiji Standard Time", "Pacific/Fiji", FLB_TZ_UTC_OFFSET(12, 0) }, + { "Chatham Islands Standard Time", "Pacific/Chatham", FLB_TZ_UTC_OFFSET(12, 45) }, + { "UTC+13", "Etc/GMT-13", FLB_TZ_UTC_OFFSET(13, 0) }, + { "UTC+13", "Pacific/Enderbury", FLB_TZ_UTC_OFFSET(13, 0) }, + { "UTC+13", "Pacific/Fakaofo", FLB_TZ_UTC_OFFSET(13, 0) }, + { "Tonga Standard Time", "Pacific/Tongatapu", FLB_TZ_UTC_OFFSET(13, 0) }, + { "Samoa Standard Time", "Pacific/Apia", FLB_TZ_UTC_OFFSET(13, 0) }, + { "Line Islands Standard Time", "Pacific/Kiritimati", FLB_TZ_UTC_OFFSET(14, 0) }, + { "Line Islands Standard Time", "Etc/GMT-14", FLB_TZ_UTC_OFFSET(14, 0) }, + { NULL, NULL, 0 } +}; + + +static pthread_once_t flb_time_tz_once = PTHREAD_ONCE_INIT; + +static size_t tz_by_windows_count = 0; +static size_t tz_by_iana_count = 0; + +static const struct flb_time_tz_map *tz_by_windows[sizeof(windows_iana_timezones) / sizeof(windows_iana_timezones[0])]; +static const struct flb_time_tz_map *tz_by_iana[sizeof(windows_iana_timezones) / sizeof(windows_iana_timezones[0])]; + +static int compare_windows(const void *a, const void *b) +{ + const struct flb_time_tz_map *entry_a = *(const struct flb_time_tz_map **)a; + const struct flb_time_tz_map *entry_b = *(const struct flb_time_tz_map **)b; + return strcasecmp(entry_a->windows, entry_b->windows); +} + +static int compare_iana(const void *a, const void *b) +{ + const struct flb_time_tz_map *entry_a = *(const struct flb_time_tz_map **)a; + const struct flb_time_tz_map *entry_b = *(const struct flb_time_tz_map **)b; + return strcmp(entry_a->iana, entry_b->iana); +} + +static int compare_search_windows(const void *key, const void *member) +{ + const char *windows_key = (const char *)key; + const struct flb_time_tz_map *entry = *(const struct flb_time_tz_map **)member; + return strcasecmp(windows_key, entry->windows); +} + +static int compare_search_iana(const void *key, const void *member) +{ + const char *iana_key = (const char *)key; + const struct flb_time_tz_map *entry = *(const struct flb_time_tz_map **)member; + return strcmp(iana_key, entry->iana); +} + +static void flb_time_tz_init(void) +{ + int i, j; + int found; + + tz_by_windows_count = 0; + tz_by_iana_count = 0; + + for (i = 0; windows_iana_timezones[i].windows != NULL; i++) { + /* Add to Windows table if not already present (keeps the first/canonical entry) */ + found = 0; + for (j = 0; j < tz_by_windows_count; j++) { + if (strcasecmp(windows_iana_timezones[i].windows, tz_by_windows[j]->windows) == 0) { + found = 1; + break; + } + } + if (!found) { + tz_by_windows[tz_by_windows_count++] = &windows_iana_timezones[i]; + } + + /* IANA zones are unique in the table, so we just add all of them */ + tz_by_iana[tz_by_iana_count++] = &windows_iana_timezones[i]; + } + + /* Sort the arrays */ + qsort(tz_by_windows, tz_by_windows_count, sizeof(tz_by_windows[0]), compare_windows); + qsort(tz_by_iana, tz_by_iana_count, sizeof(tz_by_iana[0]), compare_iana); +} + +const char *flb_time_windows_zone_to_iana(const char *windows_zone) +{ + const struct flb_time_tz_map **res; + + if (windows_zone == NULL) { + return NULL; + } + + pthread_once(&flb_time_tz_once, flb_time_tz_init); + + res = bsearch(windows_zone, tz_by_windows, tz_by_windows_count, + sizeof(tz_by_windows[0]), compare_search_windows); + if (res != NULL) { + return (*res)->iana; + } + + return NULL; +} + +const char *flb_time_iana_zone_to_windows(const char *iana_zone) +{ + const struct flb_time_tz_map **res; + + if (iana_zone == NULL) { + return NULL; + } + + pthread_once(&flb_time_tz_once, flb_time_tz_init); + + res = bsearch(iana_zone, tz_by_iana, tz_by_iana_count, + sizeof(tz_by_iana[0]), compare_search_iana); + if (res != NULL) { + return (*res)->windows; + } + + return NULL; +} + +int flb_time_windows_zone_to_utc_offset(const char *windows_zone, long *offset) +{ + const struct flb_time_tz_map **res; + + if (windows_zone == NULL || offset == NULL) { + return -1; + } + + pthread_once(&flb_time_tz_once, flb_time_tz_init); + + res = bsearch(windows_zone, tz_by_windows, tz_by_windows_count, + sizeof(tz_by_windows[0]), compare_search_windows); + if (res != NULL) { + *offset = (*res)->utc_offset; + return 0; + } + + return -1; +} + +int flb_time_iana_zone_to_utc_offset(const char *iana_zone, long *offset) +{ + const struct flb_time_tz_map **res; + + if (iana_zone == NULL || offset == NULL) { + return -1; + } + + pthread_once(&flb_time_tz_once, flb_time_tz_init); + + res = bsearch(iana_zone, tz_by_iana, tz_by_iana_count, + sizeof(tz_by_iana[0]), compare_search_iana); + if (res != NULL) { + *offset = (*res)->utc_offset; + return 0; + } + + return -1; +} diff --git a/src/flb_utils.c b/src/flb_utils.c index 659411619ee..1fcd472e9ef 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -812,8 +812,8 @@ int flb_utils_time_split(const char *time, int *sec, long *nsec) void flb_utils_bytes_to_human_readable_size(size_t bytes, char *out_buf, size_t size) { - unsigned long i; - unsigned long u = 1024; + uint64_t i; + uint64_t u = 1024; static const char *__units[] = { "b", "K", "M", "G", "T", "P", "E", "Z", "Y", NULL diff --git a/src/flb_zstd.c b/src/flb_zstd.c index 116fee2158f..2c59eb0c20e 100644 --- a/src/flb_zstd.c +++ b/src/flb_zstd.c @@ -28,7 +28,8 @@ struct flb_zstd_decompression_context { ZSTD_DCtx *dctx; }; -#define FLB_ZSTD_DEFAULT_CHUNK 64 * 1024 /* 64 KB buffer */ +#define FLB_ZSTD_DEFAULT_CHUNK (64 * 1024) /* 64 KB buffer */ +#define FLB_ZSTD_DECOMPRESS_MAX (100 * 1024 * 1024) /* 100 MB limit */ int flb_zstd_compress(void *in_data, size_t in_len, void **out_data, size_t *out_len) { @@ -104,7 +105,16 @@ static int zstd_uncompress_unknown_size(void *in_data, size_t in_len, void **out /* check if we need more space */ if (output.pos == out_size) { + if (out_size >= FLB_ZSTD_DECOMPRESS_MAX) { + flb_error("[zstd] maximum decompression size reached (~100 MB)"); + flb_free(buf); + ZSTD_freeDCtx(dctx); + return -1; + } out_size *= 2; + if (out_size > FLB_ZSTD_DECOMPRESS_MAX) { + out_size = FLB_ZSTD_DECOMPRESS_MAX; + } tmp = flb_realloc(buf, out_size); if (!tmp) { flb_errno(); @@ -146,6 +156,12 @@ int flb_zstd_uncompress(void *in_data, size_t in_len, void **out_data, size_t *o return ret; } + if (size > FLB_ZSTD_DECOMPRESS_MAX) { + flb_error("[zstd] maximum decompression size is %d bytes", + FLB_ZSTD_DECOMPRESS_MAX); + return -1; + } + buf = flb_malloc(size); if (!buf) { flb_errno(); diff --git a/src/http_server/flb_http_server_http2.c b/src/http_server/flb_http_server_http2.c index 295c5e1be80..d20ada6980a 100644 --- a/src/http_server/flb_http_server_http2.c +++ b/src/http_server/flb_http_server_http2.c @@ -74,6 +74,43 @@ static inline size_t http2_lower_value(size_t left_value, size_t right_value) return right_value; } +static int http2_request_body_limit_exceeded(struct flb_http_stream *stream, + size_t append_length) +{ + size_t current_length; + struct flb_http_server_session *parent_session; + struct flb_http_server *server; + size_t maximum_size; + + parent_session = (struct flb_http_server_session *) stream->parent; + + if (parent_session == NULL) { + return FLB_TRUE; + } + + server = parent_session->parent; + + if (server == NULL) { + return FLB_TRUE; + } + + maximum_size = flb_http_server_get_buffer_max_size(server); + + if (stream->request.body == NULL) { + current_length = 0; + } + else { + current_length = cfl_sds_len(stream->request.body); + } + + if (append_length > maximum_size || + current_length > maximum_size - append_length) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + /* RESPONSE */ struct flb_http_response *flb_http2_response_begin( @@ -510,6 +547,13 @@ static int http2_header_callback(nghttp2_session *inner_session, temporary_buffer[sizeof(temporary_buffer) - 1] = '\0'; stream->request.content_length = strtoull(temporary_buffer, NULL, 10); + + if (http2_request_body_limit_exceeded(stream, + stream->request.content_length)) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } } result = flb_http_request_set_header(&stream->request, @@ -657,6 +701,12 @@ static int http2_data_chunk_recv_callback(nghttp2_session *inner_session, return -1; } + if (http2_request_body_limit_exceeded(stream, len)) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + if (stream->request.body == NULL) { stream->request.body = cfl_sds_create_size(len); diff --git a/src/multiline/flb_ml.c b/src/multiline/flb_ml.c index 2316b9889a9..7f498dc882d 100644 --- a/src/multiline/flb_ml.c +++ b/src/multiline/flb_ml.c @@ -65,7 +65,6 @@ static uint64_t time_ms_now() return ms; } - int flb_ml_flush_stdout(struct flb_ml_parser *parser, struct flb_ml_stream *mst, void *data, char *buf_data, size_t buf_size) @@ -342,7 +341,7 @@ static int package_content(struct flb_ml_stream *mst, } if (processed && metadata != NULL) { - msgpack_pack_object(&stream_group->mp_md_pck, *metadata); + flb_ml_stream_group_add_metadata(stream_group, metadata); } if (truncated) { @@ -850,6 +849,9 @@ int flb_ml_append_object(struct flb_ml *ml, } st_group = flb_ml_stream_group_get(mst->parser, mst, NULL); + + /* Standalone record must preserve metadata too */ + flb_ml_stream_group_add_metadata(st_group, metadata); flb_ml_register_context(st_group, tm, obj); flb_ml_flush_stream_group(parser_i->ml_parser, mst, st_group, FLB_FALSE); } @@ -1199,6 +1201,303 @@ void flb_deduplication_list_purge(struct cfl_list *deduplication_list) } } +/* + * Metadata handling + * ----------------- + * + * Multiline needs to retain metadata until flush time. The previous + * implementation packed metadata into a msgpack_sbuffer and later unpacked + * it again on flush. For OTLP metadata this is both hot and unnecessary. + * + * We keep deep-copied msgpack_object snapshots per line, converting any + * MSGPACK_OBJECT_BIN (e.g. trace_id/span_id) into lowercase hex strings. + */ + +static void bin_to_hex(const unsigned char *bin, size_t len, char *out) +{ + static const char hex_table[] = "0123456789abcdef"; + size_t i; + + for (i = 0; i < len; i++) { + out[i * 2] = hex_table[(bin[i] >> 4) & 0x0F]; + out[i * 2 + 1] = hex_table[bin[i] & 0x0F]; + } +} + +struct flb_ml_metadata_object_entry { + msgpack_object obj; + struct mk_list _head; +}; + +static void flb_ml_msgpack_object_destroy(msgpack_object *obj) +{ + size_t i; + + if (obj == NULL) { + return; + } + + switch (obj->type) { + case MSGPACK_OBJECT_STR: + if (obj->via.str.ptr != NULL) { + flb_free((void *) obj->via.str.ptr); + } + break; + + case MSGPACK_OBJECT_BIN: + if (obj->via.bin.ptr != NULL) { + flb_free((void *) obj->via.bin.ptr); + } + break; + + case MSGPACK_OBJECT_EXT: + if (obj->via.ext.ptr != NULL) { + flb_free((void *) obj->via.ext.ptr); + } + break; + + case MSGPACK_OBJECT_ARRAY: + if (obj->via.array.ptr != NULL) { + for (i = 0; i < obj->via.array.size; i++) { + flb_ml_msgpack_object_destroy(&obj->via.array.ptr[i]); + } + flb_free(obj->via.array.ptr); + } + break; + + case MSGPACK_OBJECT_MAP: + if (obj->via.map.ptr != NULL) { + for (i = 0; i < obj->via.map.size; i++) { + flb_ml_msgpack_object_destroy(&obj->via.map.ptr[i].key); + flb_ml_msgpack_object_destroy(&obj->via.map.ptr[i].val); + } + flb_free(obj->via.map.ptr); + } + break; + + default: + break; + } + + memset(obj, 0, sizeof(*obj)); + obj->type = MSGPACK_OBJECT_NIL; +} + +static int flb_ml_msgpack_object_deep_copy_convert(msgpack_object *dst, + const msgpack_object *src) +{ + size_t i; + void *buf; + size_t len; + const unsigned char *bin_ptr; + size_t bin_size; + + if (dst == NULL || src == NULL) { + return -1; + } + + memset(dst, 0, sizeof(*dst)); + + switch (src->type) { + case MSGPACK_OBJECT_NIL: + dst->type = MSGPACK_OBJECT_NIL; + break; + + case MSGPACK_OBJECT_BOOLEAN: + dst->type = MSGPACK_OBJECT_BOOLEAN; + dst->via.boolean = src->via.boolean; + break; + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + dst->type = src->type; + dst->via.u64 = src->via.u64; + break; + + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + dst->type = src->type; + dst->via.f64 = src->via.f64; + break; + + case MSGPACK_OBJECT_STR: + dst->type = MSGPACK_OBJECT_STR; + len = src->via.str.size; + if (len == 0) { + dst->via.str.ptr = NULL; + dst->via.str.size = 0; + } + else { + buf = flb_malloc(len); + if (buf == NULL) { + return -1; + } + memcpy(buf, src->via.str.ptr, len); + dst->via.str.ptr = (const char *) buf; + dst->via.str.size = len; + } + break; + + case MSGPACK_OBJECT_BIN: { + bin_ptr = (const unsigned char *) src->via.bin.ptr; + bin_size = src->via.bin.size; + + dst->type = MSGPACK_OBJECT_STR; + + /* 2 chars per byte (no NUL terminator in msgpack str) */ + len = bin_size * 2; + if (len == 0) { + dst->via.str.ptr = NULL; + dst->via.str.size = 0; + } + else { + buf = flb_malloc(len); + if (buf == NULL) { + return -1; + } + + bin_to_hex(bin_ptr, bin_size, (char *) buf); + + dst->via.str.ptr = (const char *) buf; + dst->via.str.size = len; + } + break; + } + + case MSGPACK_OBJECT_EXT: + dst->type = MSGPACK_OBJECT_EXT; + dst->via.ext.type = src->via.ext.type; + len = src->via.ext.size; + if (len == 0) { + dst->via.ext.ptr = NULL; + dst->via.ext.size = 0; + } + else { + buf = flb_malloc(len); + if (buf == NULL) { + return -1; + } + memcpy(buf, src->via.ext.ptr, len); + dst->via.ext.ptr = (const char *) buf; + dst->via.ext.size = len; + } + break; + + case MSGPACK_OBJECT_ARRAY: + dst->type = MSGPACK_OBJECT_ARRAY; + dst->via.array.size = src->via.array.size; + if (dst->via.array.size == 0) { + dst->via.array.ptr = NULL; + break; + } + + dst->via.array.ptr = flb_calloc(dst->via.array.size, sizeof(msgpack_object)); + if (dst->via.array.ptr == NULL) { + return -1; + } + + for (i = 0; i < dst->via.array.size; i++) { + if (flb_ml_msgpack_object_deep_copy_convert(&dst->via.array.ptr[i], + &src->via.array.ptr[i]) != 0) { + flb_ml_msgpack_object_destroy(dst); + return -1; + } + } + break; + + case MSGPACK_OBJECT_MAP: + dst->type = MSGPACK_OBJECT_MAP; + dst->via.map.size = src->via.map.size; + if (dst->via.map.size == 0) { + dst->via.map.ptr = NULL; + break; + } + + dst->via.map.ptr = flb_calloc(dst->via.map.size, sizeof(msgpack_object_kv)); + if (dst->via.map.ptr == NULL) { + return -1; + } + + for (i = 0; i < dst->via.map.size; i++) { + if (flb_ml_msgpack_object_deep_copy_convert(&dst->via.map.ptr[i].key, + &src->via.map.ptr[i].key) != 0) { + flb_ml_msgpack_object_destroy(dst); + return -1; + } + + if (flb_ml_msgpack_object_deep_copy_convert(&dst->via.map.ptr[i].val, + &src->via.map.ptr[i].val) != 0) { + flb_ml_msgpack_object_destroy(dst); + return -1; + } + } + break; + + default: + /* Scalars or types we don't explicitly manage */ + *dst = *src; + break; + } + + return 0; +} + +static void flb_ml_stream_group_metadata_list_init(struct flb_ml_stream_group *group) +{ + if (group->metadata_objects_initialized == FLB_FALSE) { + mk_list_init(&group->metadata_objects); + group->metadata_objects_initialized = FLB_TRUE; + } +} + +void flb_ml_stream_group_purge_metadata(struct flb_ml_stream_group *group) +{ + struct mk_list *head; + struct mk_list *tmp; + struct flb_ml_metadata_object_entry *entry; + + if (group == NULL || group->metadata_objects_initialized == FLB_FALSE) { + return; + } + + mk_list_foreach_safe(head, tmp, &group->metadata_objects) { + entry = mk_list_entry(head, struct flb_ml_metadata_object_entry, _head); + + mk_list_del(&entry->_head); + flb_ml_msgpack_object_destroy(&entry->obj); + flb_free(entry); + } +} + +int flb_ml_stream_group_add_metadata(struct flb_ml_stream_group *group, + msgpack_object *metadata) +{ + struct flb_ml_metadata_object_entry *entry; + + if (group == NULL || metadata == NULL) { + return -1; + } + + flb_ml_stream_group_metadata_list_init(group); + + entry = flb_calloc(1, sizeof(*entry)); + if (entry == NULL) { + return -1; + } + + mk_list_entry_init(&entry->_head); + + if (flb_ml_msgpack_object_deep_copy_convert(&entry->obj, metadata) != 0) { + flb_ml_msgpack_object_destroy(&entry->obj); + flb_free(entry); + return -1; + } + + mk_list_add(&entry->_head, &group->metadata_objects); + + return 0; +} + int flb_ml_flush_metadata_buffer(struct flb_ml_stream *mst, struct flb_ml_stream_group *group, int deduplicate_metadata) @@ -1206,8 +1505,8 @@ int flb_ml_flush_metadata_buffer(struct flb_ml_stream *mst, int append_metadata_entry; cfl_hash_64bits_t metadata_entry_hash; struct cfl_list deduplication_list; - msgpack_unpacked metadata_map; - size_t offset; + struct mk_list *head; + struct flb_ml_metadata_object_entry *entry; size_t index; msgpack_object value; msgpack_object key; @@ -1215,78 +1514,71 @@ int flb_ml_flush_metadata_buffer(struct flb_ml_stream *mst, ret = FLB_EVENT_ENCODER_SUCCESS; + if (group == NULL || + group->metadata_objects_initialized == FLB_FALSE || + (group->metadata_objects.next == &group->metadata_objects)) { + return ret; + } + if (deduplicate_metadata) { flb_deduplication_list_init(&deduplication_list); } - msgpack_unpacked_init(&metadata_map); + mk_list_foreach(head, &group->metadata_objects) { + entry = mk_list_entry(head, struct flb_ml_metadata_object_entry, _head); - offset = 0; - while (ret == FLB_EVENT_ENCODER_SUCCESS && - msgpack_unpack_next(&metadata_map, - group->mp_md_sbuf.data, - group->mp_md_sbuf.size, - &offset) == MSGPACK_UNPACK_SUCCESS) { + if (entry->obj.type != MSGPACK_OBJECT_MAP) { + continue; + } for (index = 0; - index < metadata_map.data.via.map.size && + index < entry->obj.via.map.size && ret == FLB_EVENT_ENCODER_SUCCESS; index++) { - key = metadata_map.data.via.map.ptr[index].key; - value = metadata_map.data.via.map.ptr[index].val; + + key = entry->obj.via.map.ptr[index].key; + value = entry->obj.via.map.ptr[index].val; append_metadata_entry = FLB_TRUE; if (deduplicate_metadata) { - ret = flb_hash_msgpack_object_list(&metadata_entry_hash, - 2, - &key, - &value); - if (ret != 0) { + if (flb_hash_msgpack_object_list(&metadata_entry_hash, + 2, + &key, + &value) != 0) { ret = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; + break; } - else { - ret = flb_deduplication_list_validate( - &deduplication_list, - metadata_entry_hash); - - if (ret) { - append_metadata_entry = FLB_FALSE; - ret = FLB_EVENT_ENCODER_SUCCESS; - } - else { - ret = flb_deduplication_list_add( - &deduplication_list, - metadata_entry_hash); - - if (ret == 0) { - ret = FLB_EVENT_ENCODER_SUCCESS; - } - else { - ret = FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR; - } + if (flb_deduplication_list_validate(&deduplication_list, + metadata_entry_hash)) { + append_metadata_entry = FLB_FALSE; + } + else { + if (flb_deduplication_list_add(&deduplication_list, + metadata_entry_hash) != 0) { + ret = FLB_EVENT_ENCODER_ERROR_ALLOCATION_ERROR; + break; } } } if (append_metadata_entry) { - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_append_metadata_values( - &mst->ml->log_event_encoder, - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&key), - FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&value)); - } + ret = flb_log_event_encoder_append_metadata_values( + &mst->ml->log_event_encoder, + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&key), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&value)); } } } - msgpack_unpacked_destroy(&metadata_map); - if (deduplicate_metadata) { flb_deduplication_list_purge(&deduplication_list); } + /* Always purge stored metadata snapshots after flush */ + flb_ml_stream_group_purge_metadata(group); + return ret; } @@ -1334,6 +1626,8 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, if (ret != MSGPACK_UNPACK_SUCCESS) { flb_error("[multiline] could not unpack first line state buffer"); msgpack_unpacked_destroy(&result); + msgpack_sbuffer_destroy(&mp_sbuf); + flb_ml_stream_group_purge_metadata(group); group->mp_sbuf.size = 0; return -1; } @@ -1342,6 +1636,8 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, if (map.type != MSGPACK_OBJECT_MAP) { flb_error("[multiline] expected MAP type in first line state buffer"); msgpack_unpacked_destroy(&result); + msgpack_sbuffer_destroy(&mp_sbuf); + flb_ml_stream_group_purge_metadata(group); group->mp_sbuf.size = 0; return -1; } @@ -1461,6 +1757,10 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, if (ret != FLB_EVENT_ENCODER_SUCCESS) { flb_error("[multiline] error packing event"); + flb_ml_stream_group_purge_metadata(group); + group->mp_md_sbuf.size = 0; + msgpack_sbuffer_destroy(&mp_sbuf); + return -1; } @@ -1485,6 +1785,7 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, msgpack_sbuffer_destroy(&mp_sbuf); flb_sds_len_set(group->buf, 0); group->truncated = FLB_FALSE; + flb_ml_stream_group_purge_metadata(group); group->mp_md_sbuf.size = 0; /* Update last flush time */ diff --git a/src/multiline/flb_ml_stream.c b/src/multiline/flb_ml_stream.c index 2d48582780b..08c5ea4a35a 100644 --- a/src/multiline/flb_ml_stream.c +++ b/src/multiline/flb_ml_stream.c @@ -147,6 +147,7 @@ static void stream_group_destroy(struct flb_ml_stream_group *group) msgpack_sbuffer_destroy(&group->mp_sbuf); mk_list_del(&group->_head); + flb_ml_stream_group_purge_metadata(group); flb_free(group); } diff --git a/src/unicode/flb_utf8_and_win.c b/src/unicode/flb_utf8_and_win.c index 9d5a5c3382d..28f104924b3 100644 --- a/src/unicode/flb_utf8_and_win.c +++ b/src/unicode/flb_utf8_and_win.c @@ -118,7 +118,7 @@ struct flb_unicode_converter win866_converter = { .aliases = {"CP866", NULL}, .desc = "Windows code pages' converters", .encoding = FLB_WIN866, - .max_width = 2, + .max_width = 3, .cb_to_utf8 = flb_win_to_utf8, .cb_from_utf8 = flb_utf8_to_win, }; diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index a9ab28649f0..ce0293ac82c 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -37,6 +37,7 @@ set(UNIT_TESTS_FILES flb_event_loop.c ring_buffer.c regex.c + scheduler.c parser_json.c parser_ltsv.c parser_regex.c diff --git a/tests/internal/avro.c b/tests/internal/avro.c index 45f0734391c..7fca32c6c32 100644 --- a/tests/internal/avro.c +++ b/tests/internal/avro.c @@ -1,5 +1,6 @@ /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ #include +#include #include #include #include @@ -24,38 +25,44 @@ const char JSON_SINGLE_MAP_001_SCHEMA[] = {\"type\": \"array\", \"items\":\ {\"type\": \"map\",\"values\": \"int\"}}}]}"; -msgpack_unpacked test_init(avro_value_t *aobject, avro_schema_t *aschema, const char *json_schema, const char *json_data) { - char *out_buf; +msgpack_unpacked test_init(avro_value_t *aobject, avro_schema_t *aschema, + const char *json_schema, const char *json_data, + char **out_buf) +{ size_t out_size; int root_type; + size_t len; + char *data; + avro_value_iface_t *aclass; + msgpack_unpacked msg; - avro_value_iface_t *aclass = flb_avro_init(aobject, (char *)json_schema, strlen(json_schema), aschema); + aclass = flb_avro_init(aobject, (char *)json_schema, strlen(json_schema), aschema); TEST_CHECK(aclass != NULL); - char *data = mk_file_to_buffer(json_data); + data = mk_file_to_buffer(json_data); TEST_CHECK(data != NULL); - size_t len = strlen(data); + len = strlen(data); - TEST_CHECK(flb_pack_json(data, len, &out_buf, &out_size, &root_type, NULL) == 0); + TEST_CHECK(flb_pack_json(data, len, out_buf, &out_size, &root_type, NULL) == 0); - msgpack_unpacked msg; msgpack_unpacked_init(&msg); - TEST_CHECK(msgpack_unpack_next(&msg, out_buf, out_size, NULL) == MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(msgpack_unpack_next(&msg, *out_buf, out_size, NULL) == MSGPACK_UNPACK_SUCCESS); avro_value_iface_decref(aclass); flb_free(data); - flb_free(out_buf); return msg; } /* Unpack msgpack per avro schema */ void test_unpack_to_avro() { + char *out_buf = NULL; avro_value_t aobject; avro_schema_t aschema; + msgpack_unpacked mp; - msgpack_unpacked mp = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA, AVRO_SINGLE_MAP1); + mp = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA, AVRO_SINGLE_MAP1, &out_buf); msgpack_object_print(stderr, mp.data); flb_msgpack_to_avro(&aobject, &mp.data); @@ -141,6 +148,7 @@ void test_unpack_to_avro() avro_value_decref(&aobject); avro_schema_decref(aschema); msgpack_unpacked_destroy(&mp); + flb_free(out_buf); } void test_parse_reordered_schema() @@ -154,11 +162,12 @@ void test_parse_reordered_schema() int i=0; for (i=0; schemas[i] != NULL ; i++) { - + char *out_buf = NULL; avro_value_t aobject = {0}; avro_schema_t aschema = {0}; + msgpack_unpacked msg; - msgpack_unpacked msg = test_init(&aobject, &aschema, schemas[i], AVRO_MULTILINE_JSON); + msg = test_init(&aobject, &aschema, schemas[i], AVRO_MULTILINE_JSON, &out_buf); msgpack_object_print(stderr, msg.data); @@ -226,6 +235,7 @@ void test_parse_reordered_schema() avro_schema_decref(aschema); msgpack_unpacked_destroy(&msg); avro_value_decref(&aobject); + flb_free(out_buf); } } @@ -265,6 +275,151 @@ void test_msgpack2avro() msgpack_zone_destroy(&mempool); msgpack_sbuffer_destroy(&sbuf); } + +const char JSON_INT64_SCHEMA[] = +"{\"type\":\"record\"," +"\"name\":\"Int64Record\"," +"\"fields\":[" +"{\"name\":\"positive\",\"type\":\"long\"}," +"{\"name\":\"negative\",\"type\":\"long\"}]}"; + +const char JSON_INT_SCHEMA[] = +"{\"type\":\"record\"," +"\"name\":\"IntRecord\"," +"\"fields\":[" +"{\"name\":\"bad\",\"type\":\"int\"}]}"; + +const char JSON_INT_MULTI_FIELD_SCHEMA[] = +"{\"type\":\"record\"," +"\"name\":\"IntMultiFieldRecord\"," +"\"fields\":[" +"{\"name\":\"first\",\"type\":\"int\"}," +"{\"name\":\"bad\",\"type\":\"int\"}," +"{\"name\":\"last\",\"type\":\"string\"}]}"; + +void test_msgpack_to_avro_int64() +{ + int64_t positive_expected = 4294967296LL; + int64_t negative_expected = -2147483649LL; + int64_t actual = 0; + avro_value_t aobject; + avro_value_t test_value; + avro_schema_t aschema; + avro_value_iface_t *aclass; + msgpack_sbuffer sbuf; + msgpack_packer pk; + msgpack_unpacked msg; + + aclass = flb_avro_init(&aobject, (char *) JSON_INT64_SCHEMA, + strlen(JSON_INT64_SCHEMA), &aschema); + TEST_CHECK(aclass != NULL); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&pk, 2); + msgpack_pack_str(&pk, 8); + msgpack_pack_str_body(&pk, "positive", 8); + msgpack_pack_uint64(&pk, (uint64_t) positive_expected); + msgpack_pack_str(&pk, 8); + msgpack_pack_str_body(&pk, "negative", 8); + msgpack_pack_int64(&pk, negative_expected); + + msgpack_unpacked_init(&msg); + TEST_CHECK(msgpack_unpack_next(&msg, sbuf.data, sbuf.size, NULL) == + MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(flb_msgpack_to_avro(&aobject, &msg.data) == FLB_TRUE); + + TEST_CHECK(avro_value_get_by_name(&aobject, "positive", &test_value, NULL) == 0); + TEST_CHECK(avro_value_get_long(&test_value, &actual) == 0); + TEST_CHECK(actual == positive_expected); + + TEST_CHECK(avro_value_get_by_name(&aobject, "negative", &test_value, NULL) == 0); + TEST_CHECK(avro_value_get_long(&test_value, &actual) == 0); + TEST_CHECK(actual == negative_expected); + + msgpack_unpacked_destroy(&msg); + msgpack_sbuffer_destroy(&sbuf); + avro_value_decref(&aobject); + avro_value_iface_decref(aclass); + avro_schema_decref(aschema); +} + +void test_msgpack_to_avro_int_range() +{ + uint64_t too_big = 2147483648ULL; + avro_value_t aobject; + avro_schema_t aschema; + avro_value_iface_t *aclass; + msgpack_sbuffer sbuf; + msgpack_packer pk; + msgpack_unpacked msg; + + aclass = flb_avro_init(&aobject, (char *) JSON_INT_SCHEMA, + strlen(JSON_INT_SCHEMA), &aschema); + TEST_CHECK(aclass != NULL); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&pk, 1); + msgpack_pack_str(&pk, 3); + msgpack_pack_str_body(&pk, "bad", 3); + msgpack_pack_uint64(&pk, too_big); + + msgpack_unpacked_init(&msg); + TEST_CHECK(msgpack_unpack_next(&msg, sbuf.data, sbuf.size, NULL) == + MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(flb_msgpack_to_avro(&aobject, &msg.data) == FLB_FALSE); + + msgpack_unpacked_destroy(&msg); + msgpack_sbuffer_destroy(&sbuf); + avro_value_decref(&aobject); + avro_value_iface_decref(aclass); + avro_schema_decref(aschema); +} + +void test_msgpack_to_avro_int_range_multi_field() +{ + uint64_t too_big = 2147483648ULL; + avro_value_t aobject; + avro_schema_t aschema; + avro_value_iface_t *aclass; + msgpack_sbuffer sbuf; + msgpack_packer pk; + msgpack_unpacked msg; + + aclass = flb_avro_init(&aobject, (char *) JSON_INT_MULTI_FIELD_SCHEMA, + strlen(JSON_INT_MULTI_FIELD_SCHEMA), &aschema); + TEST_CHECK(aclass != NULL); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_map(&pk, 3); + msgpack_pack_str(&pk, 5); + msgpack_pack_str_body(&pk, "first", 5); + msgpack_pack_int(&pk, 1); + msgpack_pack_str(&pk, 3); + msgpack_pack_str_body(&pk, "bad", 3); + msgpack_pack_uint64(&pk, too_big); + msgpack_pack_str(&pk, 4); + msgpack_pack_str_body(&pk, "last", 4); + msgpack_pack_str(&pk, 1); + msgpack_pack_str_body(&pk, "x", 1); + + msgpack_unpacked_init(&msg); + TEST_CHECK(msgpack_unpack_next(&msg, sbuf.data, sbuf.size, NULL) == + MSGPACK_UNPACK_SUCCESS); + TEST_CHECK(flb_msgpack_to_avro(&aobject, &msg.data) == FLB_FALSE); + + msgpack_unpacked_destroy(&msg); + msgpack_sbuffer_destroy(&sbuf); + avro_value_decref(&aobject); + avro_value_iface_decref(aclass); + avro_schema_decref(aschema); +} + const char JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION[] = "{\"type\":\"record\",\ \"name\":\"Map001\",\ @@ -282,10 +437,12 @@ const char JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION[] = {\"type\": \"map\",\"values\": \"int\"}}}]}"; void test_union_type_sanity() { + char *out_buf = NULL; avro_value_t aobject; avro_schema_t aschema; + msgpack_unpacked msg; - msgpack_unpacked msg = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION, AVRO_SINGLE_MAP1); + msg = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION, AVRO_SINGLE_MAP1, &out_buf); msgpack_object_print(stderr, msg.data); flb_msgpack_to_avro(&aobject, &msg.data); @@ -341,14 +498,17 @@ void test_union_type_sanity() avro_value_decref(&aobject); avro_schema_decref(aschema); msgpack_unpacked_destroy(&msg); + flb_free(out_buf); } void test_union_type_branches() { + char *out_buf = NULL; avro_value_t aobject; avro_schema_t aschema; + msgpack_unpacked mp; - msgpack_unpacked mp = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION, AVRO_SINGLE_MAP1); + mp = test_init(&aobject, &aschema, JSON_SINGLE_MAP_001_SCHEMA_WITH_UNION, AVRO_SINGLE_MAP1, &out_buf); flb_msgpack_to_avro(&aobject, &mp.data); @@ -371,11 +531,16 @@ void test_union_type_branches() avro_value_decref(&aobject); avro_schema_decref(aschema); msgpack_unpacked_destroy(&mp); + flb_free(out_buf); } TEST_LIST = { /* Avro */ { "msgpack_to_avro_basic", test_unpack_to_avro}, { "test_parse_reordered_schema", test_parse_reordered_schema}, + { "test_msgpack_to_avro_int64", test_msgpack_to_avro_int64}, + { "test_msgpack_to_avro_int_range", test_msgpack_to_avro_int_range}, + { "test_msgpack_to_avro_int_range_multi_field", + test_msgpack_to_avro_int_range_multi_field}, { "test_union_type_sanity", test_union_type_sanity}, { "test_union_type_branches", test_union_type_branches}, { 0 } diff --git a/tests/internal/aws_util.c b/tests/internal/aws_util.c index 6f6fbdb1aca..8132a5c6d59 100644 --- a/tests/internal/aws_util.c +++ b/tests/internal/aws_util.c @@ -163,6 +163,13 @@ static void test_flb_aws_endpoint() endpoint) == 0); flb_free(endpoint); + /* EU Sovereign Cloud regions have a different domain */ + endpoint = flb_aws_endpoint("cloudwatch", "eusc-de-east-1"); + + TEST_CHECK(strcmp("cloudwatch.eusc-de-east-1.amazonaws.eu", + endpoint) == 0); + flb_free(endpoint); + } static void test_flb_get_s3_key_multi_tag_exists() diff --git a/tests/internal/env.c b/tests/internal/env.c index c0a72f0a896..a3745c8815d 100644 --- a/tests/internal/env.c +++ b/tests/internal/env.c @@ -85,7 +85,163 @@ void test_translate_long_env() } +/* ${name:-word} when unset or empty */ +void test_expand_default_hyphen() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + const char *v; + int ret; + + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + return; + } + + buf = flb_env_var_translate(env, "${FLB_IT_DEFHYPH_Z:-fallback}"); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("translate failed"); + flb_env_destroy(env); + return; + } + if (!TEST_CHECK(strcmp(buf, "fallback") == 0)) { + TEST_MSG("expected fallback, got=%s", buf); + } + flb_sds_destroy(buf); + + v = flb_env_get(env, "FLB_IT_DEFHYPH_Z"); + if (!TEST_CHECK(v == NULL)) { + TEST_MSG(":- must not assign locally"); + } + + ret = flb_env_set(env, "FLB_IT_DEFHYPH_Z", ""); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set empty failed"); + flb_env_destroy(env); + return; + } + buf = flb_env_var_translate(env, "${FLB_IT_DEFHYPH_Z:-empty_fallback}"); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("translate empty failed"); + flb_env_destroy(env); + return; + } + if (!TEST_CHECK(strcmp(buf, "empty_fallback") == 0)) { + TEST_MSG("empty local should use default, got=%s", buf); + } + flb_sds_destroy(buf); + flb_env_destroy(env); +} + + +/* ${name:-} should return an empty string if name is unset or empty */ +void test_expand_empty_default() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + int ret; + + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + return; + } + + /* Case 1: Variable is unset */ + buf = flb_env_var_translate(env, "val=${VAR_NOT_SET:-}"); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("translate failed"); + flb_env_destroy(env); + return; + } + if (!TEST_CHECK(strcmp(buf, "val=") == 0)) { + TEST_MSG("expected 'val=', got='%s'", buf); + } + flb_sds_destroy(buf); + + /* Case 2: Variable is explicitly set to empty string */ + ret = flb_env_set(env, "VAR_EMPTY", ""); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set failed"); + flb_env_destroy(env); + return; + } + + buf = flb_env_var_translate(env, "${VAR_EMPTY:-}"); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("translate failed"); + flb_env_destroy(env); + return; + } + + /* Result should be length 0 */ + if (!TEST_CHECK(strlen(buf) == 0)) { + TEST_MSG("expected empty string, got='%s'", buf); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); +} + + +void test_expand_default_hyphen_from_os() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *var_name = "FLB_IT_DEFHYPH_OS"; + char *var_val = "from_process"; + int ret; + + /* Set OS environment inline */ + #ifdef FLB_SYSTEM_WINDOWS + ret = _putenv_s(var_name, var_val); + #else + ret = setenv(var_name, var_val, 1); + #endif + + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("putenv failed"); + return; + } + + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + #ifdef FLB_SYSTEM_WINDOWS + _putenv_s(var_name, ""); + #else + unsetenv(var_name); + #endif + return; + } + + buf = flb_env_var_translate(env, "${FLB_IT_DEFHYPH_OS:-fallback}"); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("translate failed"); + } + else { + if (!TEST_CHECK(strcmp(buf, var_val) == 0)) { + TEST_MSG("expected from_process, got=%s", buf); + } + flb_sds_destroy(buf); + } + + flb_env_destroy(env); + + /* Unset OS environment inline */ + #ifdef FLB_SYSTEM_WINDOWS + _putenv_s(var_name, ""); + #else + unsetenv(var_name); + #endif +} + + TEST_LIST = { { "translate_long_env" , test_translate_long_env}, + { "expand_default_hyphen" , test_expand_default_hyphen}, + { "expand_empty_default", test_expand_empty_default}, + { "expand_default_hyphen_from_os", test_expand_default_hyphen_from_os}, { NULL, NULL } }; diff --git a/tests/internal/flb_time.c b/tests/internal/flb_time.c index a4e2980b492..e39f3154f66 100644 --- a/tests/internal/flb_time.c +++ b/tests/internal/flb_time.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "flb_tests_internal.h" #define SEC_32BIT 1647061992 /* 0x622c2be8 */ @@ -323,6 +324,148 @@ void test_append_to_msgpack_eventtime() msgpack_unpacked_destroy(&result); } +void test_windows_zone_to_iana() +{ + const char *iana; + + iana = flb_time_windows_zone_to_iana("Pacific Standard Time"); + if (!TEST_CHECK(iana != NULL && strcmp(iana, "America/Los_Angeles") == 0)) { + TEST_MSG("got %s, expect America/Los_Angeles", iana); + } + + iana = flb_time_windows_zone_to_iana("arabian standard time"); + if (!TEST_CHECK(iana != NULL && strcmp(iana, "Asia/Dubai") == 0)) { + TEST_MSG("got %s, expect Asia/Dubai", iana); + } + + iana = flb_time_windows_zone_to_iana("Middle East Standard Time"); + if (!TEST_CHECK(iana != NULL && strcmp(iana, "Asia/Beirut") == 0)) { + TEST_MSG("got %s, expect Asia/Beirut", iana); + } + + iana = flb_time_windows_zone_to_iana("India Standard Time"); + if (!TEST_CHECK(iana != NULL && strcmp(iana, "Asia/Kolkata") == 0)) { + TEST_MSG("got %s, expect Asia/Kolkata", iana); + } + + iana = flb_time_windows_zone_to_iana("Nepal Standard Time"); + if (!TEST_CHECK(iana != NULL && strcmp(iana, "Asia/Kathmandu") == 0)) { + TEST_MSG("got %s, expect Asia/Kathmandu", iana); + } + + iana = flb_time_windows_zone_to_iana("Unknown Standard Time"); + if (!TEST_CHECK(iana == NULL)) { + TEST_MSG("got %s, expect NULL", iana); + } + + iana = flb_time_windows_zone_to_iana(NULL); + if (!TEST_CHECK(iana == NULL)) { + TEST_MSG("got %s, expect NULL", iana); + } +} + +void test_iana_zone_to_windows() +{ + const char *windows; + + windows = flb_time_iana_zone_to_windows("America/Vancouver"); + if (!TEST_CHECK(windows != NULL && strcmp(windows, "Pacific Standard Time") == 0)) { + TEST_MSG("got %s, expect Pacific Standard Time", windows); + } + + windows = flb_time_iana_zone_to_windows("Etc/GMT-4"); + if (!TEST_CHECK(windows != NULL && strcmp(windows, "Arabian Standard Time") == 0)) { + TEST_MSG("got %s, expect Arabian Standard Time", windows); + } + + windows = flb_time_iana_zone_to_windows("Asia/Kathmandu"); + if (!TEST_CHECK(windows != NULL && strcmp(windows, "Nepal Standard Time") == 0)) { + TEST_MSG("got %s, expect Nepal Standard Time", windows); + } + + windows = flb_time_iana_zone_to_windows("Etc/Unknown"); + if (!TEST_CHECK(windows == NULL)) { + TEST_MSG("got %s, expect NULL", windows); + } + + windows = flb_time_iana_zone_to_windows(NULL); + if (!TEST_CHECK(windows == NULL)) { + TEST_MSG("got %s, expect NULL", windows); + } +} + +void test_windows_zone_to_utc_offset() +{ + int ret; + long offset; + + ret = flb_time_windows_zone_to_utc_offset("SE Asia Standard Time", &offset); + if (!TEST_CHECK(ret == 0 && offset == 25200)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=25200", ret, offset); + } + + ret = flb_time_windows_zone_to_utc_offset("Nepal Standard Time", &offset); + if (!TEST_CHECK(ret == 0 && offset == 20700)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=20700", ret, offset); + } + + ret = flb_time_windows_zone_to_utc_offset("Newfoundland Standard Time", &offset); + if (!TEST_CHECK(ret == 0 && offset == -12600)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=-12600", ret, offset); + } + + ret = flb_time_windows_zone_to_utc_offset("Unknown Standard Time", &offset); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } + + ret = flb_time_windows_zone_to_utc_offset(NULL, &offset); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } + + ret = flb_time_windows_zone_to_utc_offset("UTC", NULL); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } +} + +void test_iana_zone_to_utc_offset() +{ + int ret; + long offset; + + ret = flb_time_iana_zone_to_utc_offset("Asia/Bangkok", &offset); + if (!TEST_CHECK(ret == 0 && offset == 25200)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=25200", ret, offset); + } + + ret = flb_time_iana_zone_to_utc_offset("Australia/Eucla", &offset); + if (!TEST_CHECK(ret == 0 && offset == 31500)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=31500", ret, offset); + } + + ret = flb_time_iana_zone_to_utc_offset("Etc/GMT+12", &offset); + if (!TEST_CHECK(ret == 0 && offset == -43200)) { + TEST_MSG("got ret=%d offset=%ld, expect ret=0 offset=-43200", ret, offset); + } + + ret = flb_time_iana_zone_to_utc_offset("Etc/Unknown", &offset); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } + + ret = flb_time_iana_zone_to_utc_offset(NULL, &offset); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } + + ret = flb_time_iana_zone_to_utc_offset("Etc/UTC", NULL); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("got ret=%d, expect ret=-1", ret); + } +} + TEST_LIST = { { "flb_time_to_nanosec" , test_to_nanosec}, { "flb_time_append_to_mpack_v1" , test_append_to_mpack_v1}, @@ -331,5 +474,9 @@ TEST_LIST = { { "msgpack_to_time_eventtime" , test_msgpack_to_time_eventtime}, { "msgpack_to_time_invalid" , test_msgpack_to_time_invalid}, { "append_to_msgpack_eventtime" , test_append_to_msgpack_eventtime}, + { "windows_zone_to_iana" , test_windows_zone_to_iana}, + { "iana_zone_to_windows" , test_iana_zone_to_windows}, + { "windows_zone_to_utc_offset" , test_windows_zone_to_utc_offset}, + { "iana_zone_to_utc_offset" , test_iana_zone_to_utc_offset}, { NULL, NULL } }; diff --git a/tests/internal/network.c b/tests/internal/network.c index ce80baaae77..b2941c81759 100644 --- a/tests/internal/network.c +++ b/tests/internal/network.c @@ -13,6 +13,7 @@ #define TEST_HOSTv4 "127.0.0.1" #define TEST_HOSTv6 "::1" +#define TEST_HOSTv6_BRACKETED "[::1]" #define TEST_PORT "41322" #define TEST_EV_CLIENT MK_EVENT_NOTIFICATION @@ -159,8 +160,30 @@ void test_ipv6_client_server() test_client_server(FLB_TRUE); } +void test_ipv6_bracketed_listen() +{ + flb_sockfd_t fd_server; + + errno = 0; + fd_server = flb_net_server(TEST_PORT, TEST_HOSTv6_BRACKETED, + FLB_NETWORK_DEFAULT_BACKLOG_SIZE, + FLB_FALSE); + + if (fd_server == -1 && errno == EAFNOSUPPORT) { + TEST_MSG("This protocol is not supported in this platform"); + return; + } + + TEST_CHECK(fd_server != -1); + + if (fd_server != -1) { + flb_socket_close(fd_server); + } +} + TEST_LIST = { { "ipv4_client_server", test_ipv4_client_server}, { "ipv6_client_server", test_ipv6_client_server}, + { "ipv6_bracketed_listen", test_ipv6_bracketed_listen}, { 0 } }; diff --git a/tests/internal/parser.c b/tests/internal/parser.c index b7b4eab876b..893b2a68448 100644 --- a/tests/internal/parser.c +++ b/tests/internal/parser.c @@ -9,6 +9,12 @@ #include #include +#include +#ifndef FLB_SYSTEM_WINDOWS +#include +#include +#include +#endif #include "flb_tests_internal.h" /* Parsers configuration */ @@ -22,6 +28,62 @@ #define isleap(y) ((y) % 4 == 0 && ((y) % 400 == 0 || (y) % 100 != 0)) #define year2sec(y) (isleap(y) ? 31622400 : 31536000) +#ifdef FLB_SYSTEM_WINDOWS +#define flb_test_tzset() _tzset() + +static int flb_test_setenv(const char *name, const char *value, int overwrite) +{ + if (overwrite == 0 && getenv(name) != NULL) { + return 0; + } + + return _putenv_s(name, value); +} + +static int flb_test_unsetenv(const char *name) +{ + return _putenv_s(name, ""); +} +#else +#define flb_test_tzset() tzset() +#define flb_test_setenv(name, value, overwrite) setenv(name, value, overwrite) +#define flb_test_unsetenv(name) unsetenv(name) +#endif + +static int flb_test_timezone_available(const char *iana_zone) +{ +#ifdef FLB_SYSTEM_WINDOWS + return FLB_TRUE; +#else + int ret; + size_t len; + char path[PATH_MAX]; + const char *tzdir; + struct stat st; + + tzdir = getenv("TZDIR"); + if (tzdir == NULL || tzdir[0] == '\0') { + tzdir = "/usr/share/zoneinfo"; + } + + ret = snprintf(path, sizeof(path), "%s/%s", tzdir, iana_zone); + if (ret < 0) { + return FLB_FALSE; + } + + len = (size_t) ret; + if (len >= sizeof(path)) { + return FLB_FALSE; + } + + if (stat(path, &st) != 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +#endif +} + /* Timezone */ struct tz_check { char *val; @@ -517,12 +579,273 @@ void test_mysql_unquoted() } +void test_parser_time_system_timezone_midnight() +{ + int ret; + time_t now; + time_t epoch; + struct tm now_tm; + struct tm expected_tm; + struct flb_tm tm; + double ns; + char *orig_tz; + struct flb_parser *parser; + struct flb_config *config; +#ifdef FLB_SYSTEM_WINDOWS + char *test_tz = "CET-1CEST,M3.5.0/2,M10.5.0/3"; +#else + char *test_tz = "Europe/Stockholm"; +#endif + + orig_tz = getenv("TZ"); + if (orig_tz != NULL) { + orig_tz = flb_strdup(orig_tz); + } + + ret = flb_test_setenv("TZ", test_tz, 1); + TEST_CHECK(ret == 0); + flb_test_tzset(); + + memset(&now_tm, 0, sizeof(struct tm)); + now_tm.tm_year = 2025 - 1900; + now_tm.tm_mon = 3; + now_tm.tm_mday = 15; + now_tm.tm_hour = 0; + now_tm.tm_min = 30; + now_tm.tm_sec = 0; + now_tm.tm_isdst = -1; + now = mktime(&now_tm); + TEST_CHECK(now != (time_t) -1); + + config = flb_config_init(); + parser = flb_parser_create("time_only_tz", "regex", "^(?