Perf/compression plages et filtrage

README.md

@@ -50,6 +50,24 @@ hashtheplanet --input src/tech_list.json --json-dir dist/ --cache-dir .git-cache
 - `--workers` sets the number of parallel threads (default: 4).
 - Existing JSON files in `--json-dir` are loaded and merged with new results, enabling incremental updates.
 
+### Pruning non-discriminating files
+
+Most static files never change across a long run of releases, so their hash cannot narrow down a
+version. Each file is scored:
+
+```
+score = number of distinct hashes / size of the largest version group
+```
+
+A file with many hashes and small version groups is discriminating (high score); a file with a
+single hash shared by 200 versions is not (score 0.005). Files scoring below
+`--discrimination-threshold` are dropped (default: `0.05`, use `0` to disable).
+
+On the current technology set this removes 25-54% of the files and 23% of the total output size,
+while only two versions across all technologies (`drupal 8.1.0-rc1` and `wordpress 5.4.1`) lose the
+ability to be pinpointed exactly. No version loses coverage entirely. Raise the threshold for a
+smaller database, lower it to keep more candidate files to probe.
+
 ### Look up a hash
 
 ```bash
@@ -63,40 +81,80 @@ hashtheplanet --file path/to/style.css --json-dir dist/
 ### All options
 
 ```
-usage: hashtheplanet [-h] [-i INPUT] [--json-dir JSON_DIR]
-                     [--cache-dir CACHE_DIR] [--workers WORKERS]
-                     [--color] [-v {DEBUG,INFO,WARNING}]
-                     [--hash HASH] [-f FILE] [--version]
+usage: hashtheplanet [-h] [-i INPUT] [--json-dir JSON_DIR] [--cache-dir CACHE_DIR]
+                     [--workers WORKERS] [--discrimination-threshold DISCRIMINATION_THRESHOLD]
+                     [--color] [-v {DEBUG,INFO,WARNING}] [--hash HASH] [-f FILE] [--version]
 
 options:
-  -i, --input          Input file (json) with resources targets
-  --json-dir           Output directory for JSON hash files (default: dist)
-  --cache-dir          Directory to cache bare git repositories
-  --workers            Number of parallel workers (default: 4)
-  --color              Colorize output
-  -v, --verbose        Set verbosity level
-  --hash               Search for a file hash in the generated JSON files
-  -f, --file           Compute git hash of a file and search for it
-  --version            Show program's version number and exit
+  -i, --input                    Input file (json) with resources targets
+  --json-dir                     Output directory for JSON hash files (default: dist)
+  --cache-dir                    Directory to cache bare git repositories
+  --workers                      Number of parallel workers (default: 4)
+  --discrimination-threshold     Remove files with a discrimination score below this
+                                 threshold (0 to disable, default: 0.05)
+  --color                        Colorize output
+  -v, --verbose                  Set verbosity level
+  --hash                         Search for a file hash in the generated JSON files
+  -f, --file                     Compute git hash of a file and search for it
+  --version                      Show program's version number and exit
 ```
 
 ## Output format
 
-Each JSON file follows the format expected by Wapiti:
+Each JSON file uses format version 2, which stores version **ranges** instead of exhaustive
+version lists:
 
 ```json
 {
-    "wp-admin/css/about.min.css": {
-        "7dca9b7fd6334608de4d196b761898faec68a22e": ["4.5", "4.5.1", "4.5.10"],
-        "bd003058c2c8e8fa9537d4079be424285170665f": ["4.5.24", "4.5.25"]
+    "_meta": {
+        "format_version": 2,
+        "sorted_versions": ["1.5", "1.5.1", "...", "4.5.23", "4.5.24", "...", "4.5.33", "..."]
     },
-    "wp-includes/js/jquery/jquery.min.js": {
-        "a1b2c3d4...": ["5.0", "5.0.1"]
+    "files": {
+        "wp-admin/css/about.min.css": {
+            "7dca9b7fd6334608de4d196b761898faec68a22e": ["4.5-4.5.23"],
+            "bd003058c2c8e8fa9537d4079be424285170665f": ["4.5.24-4.5.33"]
+        },
+        "wp-includes/js/jquery/jquery.min.js": {
+            "a1b2c3d4...": ["5.0", "5.0.1"]
+        }
     }
 }
 ```
 
-Structure: `{ file_path: { git_blob_sha1: [versions] } }`
+- `_meta.sorted_versions` is the ordered list of every version known for this technology. It is the
+  reference for range boundaries.
+- `files` maps `file_path -> { git_blob_sha1: [ranges] }`. Each entry is either a single version
+  (`"5.0"`) or an inclusive range (`"4.5-4.5.23"`) covering every consecutive entry of
+  `sorted_versions` between the two bounds.
+
+A range is only emitted for versions that are *adjacent in `sorted_versions`*, so expanding a range
+never invents a version that was not observed.
+
+Version ordering handles the tagging schemes used by the supported projects: `v` prefixes
+(`v4.3.4`), pre-releases (`10.0.0-alpha1` < `10.0.0-beta1` < `10.0.0-rc1` < `10.0.0`), Magento patch
+levels (`2.4.8-p1`), Joomla underscores (`2.5.0_RC1`) and four-segment versions (`1.5.1.1`).
+Numeric segments are compared as integers, so `2.0.10` sorts after `2.0.2`.
+
+Compared to a flat `{ file_path: { git_blob_sha1: [versions] } }` mapping, this cuts the total
+output from 190 MB to 40 MB (-79% uncompressed, -20% once gzipped) at the cost of an expansion step
+when loading.
+
+### Reading the files
+
+Consumers must expand the ranges before matching. `hashtheplanet.utils.version_utils` exposes the
+helpers used internally:
+
+```python
+from hashtheplanet.utils.version_utils import expand_ranges
+
+sorted_versions = data["_meta"]["sorted_versions"]
+versions = expand_ranges(["4.5-4.5.23"], sorted_versions)
+```
+
+Since version ranges appeared in format 2, always check `_meta.format_version` before parsing. A
+file without a `_meta` key is a legacy release using the flat mapping above; `load_json` still reads
+both, which keeps incremental updates working across the format change.
 
 ## Development
 

hashtheplanet/builders/json_builder.py

@@ -6,6 +6,16 @@
 from collections import defaultdict
 from typing import Dict, List
 
+from loguru import logger
+
+from hashtheplanet.utils.version_utils import (
+    collect_all_versions,
+    compress_to_ranges,
+    expand_ranges,
+)
+
+FORMAT_VERSION = 2
+
 
 class JsonBuilder:
     """
@@ -54,24 +64,71 @@ def merge(self, other: "JsonBuilder"):
                 for hash_value, versions in hash_dict.items():
                     self._data[technology][file_path][hash_value].extend(versions)
 
+    def compute_discrimination_scores(self, technology: str) -> Dict[str, float]:
+        """
+        For each file_path in the technology, compute a discrimination score:
+          score = num_distinct_hashes / max(len(version_list) for each hash)
+
+        Higher score = more discriminating file (many hashes, small version groups).
+        Lower score = less useful file (few hashes, large version groups).
+        """
+        tech_data = self._data.get(technology, {})
+        scores = {}
+        for file_path, hash_dict in tech_data.items():
+            num_hashes = len(hash_dict)
+            max_group_size = max((len(v) for v in hash_dict.values()), default=0)
+            scores[file_path] = num_hashes / max_group_size if max_group_size > 0 else 0
+        return scores
+
+    def filter_low_discrimination_files(self, threshold: float = 0.05):
+        """
+        Remove files with discrimination score below threshold, for all technologies.
+        """
+        for technology in list(self._data.keys()):
+            scores = self.compute_discrimination_scores(technology)
+            to_remove = [fp for fp, score in scores.items() if score < threshold]
+            if to_remove:
+                for fp in to_remove:
+                    del self._data[technology][fp]
+                logger.info(
+                    f"{technology}: removed {len(to_remove)} low-discrimination files "
+                    f"(threshold={threshold}), {len(self._data[technology])} files remaining"
+                )
+
     def save_json(self, output_dir: str):
         """
         Save one JSON file per technology in the output directory.
-        Format: {file_path: {hash: [versions]}}
+        Format v2: {_meta: {format_version, sorted_versions}, files: {file_path: {hash: [ranges]}}}
         """
         os.makedirs(output_dir, exist_ok=True)
 
         for technology, tech_data in self._data.items():
             output_path = os.path.join(output_dir, f"{technology.lower()}_hash_files.json")
-            serializable = {
-                fp: dict(hashes) for fp, hashes in tech_data.items()
+
+            sorted_versions = collect_all_versions(tech_data)
+
+            files_data = {}
+            for fp, hashes in tech_data.items():
+                files_data[fp] = {}
+                for hash_value, versions in hashes.items():
+                    deduped = list(dict.fromkeys(versions))
+                    files_data[fp][hash_value] = compress_to_ranges(deduped, sorted_versions)
+
+            output = {
+                "_meta": {
+                    "format_version": FORMAT_VERSION,
+                    "sorted_versions": sorted_versions,
+                },
+                "files": files_data,
             }
+
             with open(output_path, "w", encoding="utf-8") as file_fp:
-                json.dump(serializable, file_fp, indent=4)
+                json.dump(output, file_fp, indent=4)
 
     def load_json(self, output_dir: str):
         """
         Load existing JSON files from the output directory to support incremental updates.
+        Supports both v2 format (with ranges) and legacy format.
         """
         if not os.path.isdir(output_dir):
             return
@@ -89,6 +146,15 @@ def load_json(self, output_dir: str):
             if technology not in self._data:
                 self._data[technology] = defaultdict(lambda: defaultdict(list))
 
-            for file_path, hash_dict in data.items():
-                for hash_value, versions in hash_dict.items():
-                    self._data[technology][file_path][hash_value].extend(versions)
+            if "_meta" in data and "files" in data:
+                # v2 format: expand ranges back to full version lists
+                sorted_versions = data["_meta"]["sorted_versions"]
+                for file_path, hash_dict in data["files"].items():
+                    for hash_value, range_list in hash_dict.items():
+                        versions = expand_ranges(range_list, sorted_versions)
+                        self._data[technology][file_path][hash_value].extend(versions)
+            else:
+                # Legacy format: direct version lists
+                for file_path, hash_dict in data.items():
+                    for hash_value, versions in hash_dict.items():
+                        self._data[technology][file_path][hash_value].extend(versions)

hashtheplanet/core/hashtheplanet.py

@@ -24,14 +24,16 @@ class HashThePlanet():
     The HashThePlanet class
     """
     def __init__(self, input_file: str, json_dir: str = "dist",
-                 cache_dir: str = None, max_workers: int = 4):
+                 cache_dir: str = None, max_workers: int = 4,
+                 discrimination_threshold: float = 0.05):
         """
         Initialisation requires an input filename (json) and an output directory.
         """
         self._input_file = input_file
         self._json_dir = json_dir
         self._cache_dir = cache_dir
         self._max_workers = max_workers
+        self._discrimination_threshold = discrimination_threshold
         self._config = Config()
 
     def _compute_single_target(self, resource_name: str, target: str, builder: JsonBuilder):
@@ -94,6 +96,9 @@ def compute_hashs(self):
                 except Exception as error:
                     logger.error(f"Error processing {target}: {error}")
 
+        if self._discrimination_threshold > 0:
+            builder.filter_low_discrimination_files(self._discrimination_threshold)
+
         builder.save_json(self._json_dir)
         logger.info(f"JSON files saved to {self._json_dir}")
         logger.info("Computing done")
@@ -150,6 +155,13 @@ def main():
         help="Number of parallel workers for processing repositories"
     )
 
+    parser.add_argument(
+        "--discrimination-threshold",
+        type=float,
+        default=0.05,
+        help="Remove files with discrimination score below this threshold (0 to disable, default: 0.05)"
+    )
+
     parser.add_argument(
         "--color",
         action="store_true",
@@ -194,7 +206,8 @@ def main():
         args.input,
         json_dir=args.json_dir,
         cache_dir=args.cache_dir,
-        max_workers=args.workers
+        max_workers=args.workers,
+        discrimination_threshold=args.discrimination_threshold,
     )
 
     if args.file is not None: