Fix S3 URL parsing for nested paths in AWS buckets (#1819)

* Add s3UrlFormat option, reformat regex

* Update parseS3Url params

* Update pmtiles_adapter.js

* Update pmtiles_adapter.js

* bump version

Author: acalcutt

CHANGELOG.md

@@ -1,10 +1,12 @@
 # tileserver-gl changelog
 
-## 5.5.0-pre.2
+## 5.5.0-pre.3
 * Add S3 support for PMTiles with multiple AWS credential profiles (https://github.com/maptiler/tileserver-gl/pull/1779) by @acalcutt
 * Create .aws directory passthrough folder in Dockerfile (https://github.com/maptiler/tileserver-gl/pull/1784) by @acalcutt
 * Update eslint to v9 (https://github.com/maptiler/tileserver-gl/pull/1473) by @acalcutt
 * Fix Renderer Crashes from Failed Fetches (https://github.com/maptiler/tileserver-gl/pull/1798) by @acalcutt
+* Add Visual Regression Tests for Static Image Overlays (https://github.com/maptiler/tileserver-gl/pull/1792) by @acalcutt
+* Fix S3 URL parsing for nested paths in AWS buckets (https://github.com/maptiler/tileserver-gl/pull/1819) by @acalcutt
 
 ## 5.4.0
 * Fix the issue where the tile URL cannot be correctly parsed with the HTTPS protocol when using an nginx proxy service (https://github.com/maptiler/tileserver-gl/pull/1578) by @dakanggo

docs/config.rst

@@ -311,6 +311,23 @@ Here are the available options for each data source:
     If not specified, uses ``AWS_REGION`` environment variable or defaults to ``us-east-1``.
     Optional, only applicable to PMTiles sources using S3 URLs.
 
+``s3UrlFormat`` (string)
+    Specifies how to interpret the S3 URL format.
+    
+    Allowed values:
+    
+    * ``aws`` - Interpret as AWS S3 (``s3://bucket/path/file.pmtiles``)
+    * ``custom`` - Interpret as custom S3 endpoint (``s3://endpoint/bucket/path/file.pmtiles``)
+    * Not specified (default) - Auto-detect based on URL pattern
+    
+    Can be specified in the URL using ``?s3UrlFormat=aws`` or in the configuration.
+    If both are specified, the configuration value takes precedence.
+    
+    Optional, only applicable to PMTiles sources using S3 URLs.
+
+.. note::
+    By default, URLs with dots in the first segment (e.g., ``s3://storage.example.com/bucket/file.pmtiles``) are treated as custom endpoints, while URLs without dots are treated as AWS S3. Use ``s3UrlFormat: "aws"`` if your AWS bucket name contains dots.
+
 .. note::
     These configuration options will be overridden by metadata in the MBTiles or PMTiles file. if corresponding properties exist in the file's metadata, you do not need to specify them in the data configuration.
 
@@ -445,6 +462,17 @@ Precedence order (highest to lowest): Configuration property ``s3Region``, URL p
 
 Precedence order (highest to lowest): Configuration property ``requestPayer``, URL parameter ``?requestPayer=true``, Default: ``false``.
 
+*S3UrlFormat* - Specifies how to interpret S3 URLs::
+
+  # URL parameter
+  "pmtiles": "s3://my.bucket.name/tiles.pmtiles?s3UrlFormat=aws"
+  
+  # Configuration property
+  "pmtiles": "s3://my.bucket.name/tiles.pmtiles",
+  "s3UrlFormat": "aws"
+
+Precedence order (highest to lowest): Configuration property ``s3UrlFormat``, URL parameter ``?s3UrlFormat=...``, Auto-detection.
+
 **Complete Configuration Examples:**
 
 Using URL parameters::
@@ -453,6 +481,9 @@ Using URL parameters::
     "us-west-tiles": {
       "pmtiles": "s3://prod-bucket/tiles.pmtiles?profile=production&region=us-west-2"
     },
+    "dotted-bucket-name": {
+      "pmtiles": "s3://my.bucket.name/tiles.pmtiles?s3UrlFormat=aws&region=us-east-1"
+    },
     "eu-requester-pays": {
       "pmtiles": "s3://bucket/tiles.pmtiles?profile=prod&region=eu-central-1&requestPayer=true"
     }
@@ -466,6 +497,11 @@ Using configuration properties (recommended)::
       "s3Profile": "production",
       "s3Region": "us-west-2"
     },
+    "dotted-bucket-name": {
+      "pmtiles": "s3://my.bucket.name/tiles.pmtiles",
+      "s3UrlFormat": "aws",
+      "s3Region": "us-east-1"
+    },
     "eu-requester-pays": {
       "pmtiles": "s3://bucket/tiles.pmtiles",
       "s3Profile": "production",
@@ -476,13 +512,17 @@ Using configuration properties (recommended)::
 
 **Using S3 in Style JSON Sources:**
 
-When referencing S3 sources from within a style JSON file, use the ``pmtiles://`` prefix with S3 URLs. You can only specify profile, region, and requestPayer using URL query parameters (configuration properties are not available in style JSON)::
+When referencing S3 sources from within a style JSON file, use the ``pmtiles://`` prefix with S3 URLs. You can specify profile, region, requestPayer, and s3UrlFormat using URL query parameters (configuration properties are not available in style JSON)::
 
   "sources": {
     "aws-tiles": {
       "url": "pmtiles://s3://my-bucket/tiles.pmtiles?profile=production",
       "type": "vector"
     },
+    "dotted-bucket": {
+      "url": "pmtiles://s3://my.bucket.name/tiles.pmtiles?s3UrlFormat=aws",
+      "type": "vector"
+    },
     "spaces-tiles": {
       "url": "pmtiles://s3://example-storage.com/my-bucket/tiles.pmtiles?region=nyc3",
       "type": "vector"

docs/usage.rst

@@ -64,6 +64,9 @@ The `--file` option supports multiple source types:
   # Requester-pays bucket
   tileserver-gl --file "s3://bucket/tiles.pmtiles?requestPayer=true"
 
+  # Bucket name with dots (force AWS S3 interpretation)
+  tileserver-gl --file "s3://my.bucket.name/tiles.pmtiles?s3UrlFormat=aws"
+
   # All options combined
   tileserver-gl --file "s3://bucket/tiles.pmtiles?profile=prod&region=us-west-2&requestPayer=true"
 
@@ -82,6 +85,8 @@ You can also use `pmtiles://` or `mbtiles://` prefixes to explicitly specify the
 .. note::
     For S3 sources, AWS credentials must be configured via environment variables, AWS credentials file (`~/.aws/credentials` on Linux/macOS or `C:\Users\USERNAME\.aws\credentials` on Windows), or IAM roles. 
 
+    The `s3UrlFormat` parameter can be set to `aws` or `custom` to override auto-detection when needed (e.g., for AWS bucket names containing dots).
+    
     **When using Docker**, the host credentials file can be mounted to the container's user home directory:
 
     ::

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "tileserver-gl",
-  "version": "5.5.0-pre.2",
+  "version": "5.5.0-pre.3",
   "description": "Map tile server for JSON GL styles - vector and server side generated raster tiles",
   "main": "src/main.js",
   "bin": "src/main.js",

package.json

@@ -17,31 +17,45 @@ class S3Source {
    * @param {string} [s3Profile] - Optional AWS credential profile name from config.
    * @param {boolean} [configRequestPayer] - Optional flag from config for requester pays buckets.
    * @param {string} [configRegion] - Optional AWS region from config.
+   * @param {string} [s3UrlFormat] - Optional S3 URL format from config: 'aws' or 'custom'.
    * @param {boolean} [verbose] - Whether to show verbose logging.
    */
   constructor(
     s3Url,
     s3Profile,
     configRequestPayer,
     configRegion,
+    s3UrlFormat,
     verbose = false,
   ) {
-    const parsed = this.parseS3Url(s3Url);
+    const parsed = this.parseS3Url(s3Url, s3UrlFormat);
     this.bucket = parsed.bucket;
     this.key = parsed.key;
     this.endpoint = parsed.endpoint;
     this.url = s3Url;
     this.verbose = verbose;
 
-    // Determine the final profile: Config takes precedence over URL
+    // Apply configuration precedence: Config > URL > Default
+    // Using || for strings (empty string = not set)
+    // Using ?? for booleans (false is valid value)
     const profile = s3Profile || parsed.profile;
-
-    // Determine requestPayer: Config takes precedence over URL
     this.requestPayer = configRequestPayer ?? parsed.requestPayer;
-
-    // Determine region: Config takes precedence over URL
     this.region = configRegion || parsed.region;
 
+    // Log precedence decisions for debugging
+    if (verbose >= 3) {
+      console.log(`S3 config precedence for ${s3Url}:`);
+      console.log(
+        `  Profile: ${s3Profile ? 'config' : parsed.profile ? 'url' : 'default'} = ${profile || 'none'}`,
+      );
+      console.log(
+        `  Region: ${configRegion ? 'config' : parsed.region !== (process.env.AWS_REGION || 'us-east-1') ? 'url' : 'env/default'} = ${this.region}`,
+      );
+      console.log(
+        `  RequestPayer: ${configRequestPayer !== undefined ? 'config' : parsed.requestPayer ? 'url' : 'default'} = ${this.requestPayer}`,
+      );
+    }
+
     // Create S3 client
     this.s3Client = this.createS3Client(
       parsed.endpoint,
@@ -54,60 +68,78 @@ class S3Source {
   /**
    * Parses various S3 URL formats into bucket, key, endpoint, region, and profile.
    * @param {string} url - The S3 URL to parse.
+   * @param {string} [s3UrlFormat] - Optional format override: 'aws' or 'custom'.
    * @returns {object} - An object containing bucket, key, endpoint, region, and profile.
    * @throws {Error} - Throws an error if the URL format is invalid.
    */
-  parseS3Url(url) {
-    // Initialize with defaults/environment
+  parseS3Url(url, s3UrlFormat) {
+    // Validate s3UrlFormat if provided
+    if (s3UrlFormat && s3UrlFormat !== 'aws' && s3UrlFormat !== 'custom') {
+      console.warn(
+        `Invalid s3UrlFormat: "${s3UrlFormat}". Must be "aws" or "custom". Using auto-detection.`,
+      );
+      s3UrlFormat = undefined;
+    }
+
     let region = process.env.AWS_REGION || 'us-east-1';
     let profile = null;
     let requestPayer = false;
 
-    const queryString = url.split('?')[1];
+    // Parse URL parameters
+    const [cleanUrl, queryString] = url.split('?');
     if (queryString) {
       const params = new URLSearchParams(queryString);
-      // Update profile if provided in url parameters
+      // URL parameters override defaults
       profile = params.get('profile') ?? profile;
-      // Update region if provided in url parameters
       region = params.get('region') ?? region;
-      // Update requestPayer if provided in url parameters
+      s3UrlFormat = s3UrlFormat ?? params.get('s3UrlFormat'); // Config overrides URL
+
       const payerVal = params.get('requestPayer');
       requestPayer = payerVal === 'true' || payerVal === '1';
     }
 
-    // Clean URL for format detection (remove trailing slashes)
-    const baseUrl = url.split('?')[0];
-    const cleanUrl = baseUrl.replace(/\/+$/, '');
+    // Helper to build result object
+    const buildResult = (endpoint, bucket, key) => ({
+      endpoint: endpoint ? `https://${endpoint}` : null,
+      bucket,
+      key,
+      region,
+      profile,
+      requestPayer,
+    });
 
-    // Format 1: s3://endpoint/bucket/key (S3-compatible storage)
-    // Example: s3://storage.example.com/mybucket/path/to/tiles.pmtile
-    const endpointMatch = cleanUrl.match(/^s3:\/\/([^/]+)\/([^/]+)\/(.+)$/);
-    if (endpointMatch) {
-      return {
-        endpoint: `https://${endpointMatch[1]}`,
-        bucket: endpointMatch[2],
-        key: endpointMatch[3],
-        region,
-        profile,
-        requestPayer,
-      };
-    }
+    // Define patterns based on format
+    const patterns = {
+      customWithDot: /^s3:\/\/([^/]*\.[^/]+)\/([^/]+)\/(.+)$/, // Auto-detect: requires dot
+      customForced: /^s3:\/\/([^/]+)\/([^/]+)\/(.+)$/, // Explicit: no dot required
+      aws: /^s3:\/\/([^/]+)\/(.+)$/,
+    };
 
-    // Format 2: s3://bucket/key (AWS S3 default)
-    // Example: s3://my-bucket/path/to/tiles.pmtiles
-    const awsMatch = cleanUrl.match(/^s3:\/\/([^/]+)\/(.+)$/);
-    if (awsMatch) {
-      return {
-        endpoint: null, // Use default AWS endpoint
-        bucket: awsMatch[1],
-        key: awsMatch[2],
-        region,
-        profile,
-        requestPayer,
-      };
+    // Match based on s3UrlFormat or auto-detect
+    let match;
+
+    if (s3UrlFormat === 'custom') {
+      match = cleanUrl.match(patterns.customForced);
+      if (match) return buildResult(match[1], match[2], match[3]);
+    } else if (s3UrlFormat === 'aws') {
+      match = cleanUrl.match(patterns.aws);
+      if (match) return buildResult(null, match[1], match[2]);
+    } else {
+      // Auto-detection: try custom (with dot) first, then AWS
+      match = cleanUrl.match(patterns.customWithDot);
+      if (match) return buildResult(match[1], match[2], match[3]);
+
+      match = cleanUrl.match(patterns.aws);
+      if (match) return buildResult(null, match[1], match[2]);
     }
 
-    throw new Error(`Invalid S3 URL format: ${url}`);
+    throw new Error(
+      `Invalid S3 URL format: ${url}\n` +
+        `Expected formats:\n` +
+        `  AWS S3: s3://bucket-name/path/to/file.pmtiles\n` +
+        `  Custom endpoint: s3://endpoint.com/bucket/path/to/file.pmtiles\n` +
+        `Use s3UrlFormat parameter to override auto-detection if needed.`,
+    );
   }
 
   /**
@@ -282,6 +314,7 @@ async function readFileBytes(fd, buffer, offset) {
  * @param {string} [s3Profile] - Optional AWS credential profile name.
  * @param {boolean} [requestPayer] - Optional flag for requester pays buckets.
  * @param {string} [s3Region] - Optional AWS region.
+ * @param {string} [s3UrlFormat] - Optional S3 URL format: 'aws' or 'custom'.
  * @param {boolean} [verbose] - Whether to show verbose logging.
  * @returns {PMTiles} - A PMTiles instance.
  */
@@ -290,6 +323,7 @@ export function openPMtiles(
   s3Profile,
   requestPayer,
   s3Region,
+  s3UrlFormat,
   verbose = 0,
 ) {
   let pmtiles = undefined;
@@ -303,6 +337,7 @@ export function openPMtiles(
       s3Profile,
       requestPayer,
       s3Region,
+      s3UrlFormat,
       verbose,
     );
     pmtiles = new PMTiles(source);

src/pmtiles_adapter.js

@@ -389,6 +389,7 @@ export const serve_data = {
         params.s3Profile,
         params.requestPayer,
         params.s3Region,
+        params.s3UrlFormat,
         verbose,
       );
       sourceType = 'pmtiles';

src/serve_data.js

@@ -1594,6 +1594,7 @@ export const serve_rendered = {
         let s3Profile;
         let requestPayer;
         let s3Region;
+        let s3UrlFormat;
         const dataInfo = dataResolver(dataId);
         if (dataInfo.inputFile) {
           inputFile = dataInfo.inputFile;
@@ -1602,6 +1603,7 @@ export const serve_rendered = {
           s3Profile = dataInfo.s3Profile;
           requestPayer = dataInfo.requestPayer;
           s3Region = dataInfo.s3Region;
+          s3UrlFormat = dataInfo.s3UrlFormat;
         } else {
           console.error(`ERROR: data "${inputFile}" not found!`);
           process.exit(1);
@@ -1622,6 +1624,7 @@ export const serve_rendered = {
             s3Profile,
             requestPayer,
             s3Region,
+            s3UrlFormat,
             verbose,
           );
           // eslint-disable-next-line security/detect-object-injection -- name is from style sources object keys

src/serve_rendered.js

@@ -290,6 +290,7 @@ async function start(opts) {
               let resolvedS3Profile;
               let resolvedRequestPayer;
               let resolvedS3Region;
+              let resolvedS3UrlFormat;
 
               for (const id of Object.keys(data)) {
                 // eslint-disable-next-line security/detect-object-injection -- id is from Object.keys of data config
@@ -327,6 +328,11 @@ async function start(opts) {
                       resolvedS3Profile = sourceData.s3Profile;
                     }
 
+                    // Get s3UrlFormat if present
+                    if (Object.hasOwn(sourceData, 's3UrlFormat')) {
+                      resolvedS3UrlFormat = sourceData.s3UrlFormat;
+                    }
+
                     // Get requestPayer if present
                     if (Object.hasOwn(sourceData, 'requestPayer')) {
                       resolvedRequestPayer = !!sourceData.requestPayer;
@@ -354,6 +360,7 @@ async function start(opts) {
                   s3Profile: undefined,
                   requestPayer: false,
                   s3Region: undefined,
+                  s3UrlFormat: undefined,
                 };
               }
 
@@ -385,6 +392,7 @@ async function start(opts) {
                 s3Profile: resolvedS3Profile,
                 requestPayer: resolvedRequestPayer,
                 s3Region: resolvedS3Region,
+                s3UrlFormat: resolvedS3UrlFormat,
               };
             },
           ),