data scraper for Changelogs

data/changelog/releases/dune/2024-06-17-dune.3.16.0.md

@@ -1,6 +1,7 @@
 ---
 title: Dune 3.16.0
 tags: [dune, platform]
+versions: [3.16.0, "3.16.0~alpha2", "3.16.0~alpha1"]
 changelog: |
    ### Added
 

data/changelog/releases/dune/2024-11-27-dune.3.17.0.md

@@ -1,6 +1,7 @@
 ---
 title: Dune 3.17.0
 tags: [dune, platform]
+versions: [3.17.0, "3.17.0~alpha0"]
 changelog: |
    ### Fixed
 

data/changelog/releases/ocaml-lsp/2024-12-23-ocaml-lsp-1.20.1.md

@@ -1,6 +1,7 @@
 ---
 title: OCaml-LSP 1.20.1
 tags: [ocaml-lsp, platform]
+versions: ["1.20.1", "1.20.1-4.14", "1.20.0-4.14"]
 changelog: |
     ## Features  
     - Add custom `ocamllsp/typeSearch` request ([#1369](https://github.com/ocaml/ocaml-lsp/pull/1369))  

data/changelog/releases/ocaml/2024-05-13-ocaml-5.2.0.md

@@ -2,6 +2,7 @@
 title: Release of OCaml 5.2.0
 description: Release of OCaml 5.2.0
 tags: [ocaml]
+versions: ["OCaml 5.2.0"]
 changelog: |
 
   (Changes that can break existing programs are marked with a "*")
@@ -902,7 +903,7 @@
 release of OCaml version 5.2.0.
 
 Some of the highlights in OCaml 5.2.0 are:
 - Reintroduced GC compaction
 - Restored native backend for POWER 64 bits
 - Thread sanitiser support
 - New Dynarray module
@@ -912,17 +913,17 @@
 - Local open in type expressions
 
 And a lot of incremental changes:
 - Around 20 new functions in the standard library
 - Many fixes and improvements in the runtime
 - Many bug fixes
 
 OCaml 5.2.0 is still a somewhat experimental release compared to the OCaml
 4.14 branch. In particular:
 
    - The Windows MSVC port is still unavailable.
    - Ephemeron performances need to be investigated.
    - `statmemprof` is being tested in the developer branch.
    - There are a number of known runtime concurrency or GC performance bugs
      (that trigger under rare circumstances).
 
 Since the Windows MSVC port and `statmemprof` are still missing, the maintenance
@@ -932,7 +933,7 @@
 and post any questions or comments you might have on our
 [discussion forums](https://discuss.ocaml.org).
 
 
 The full list of changes can be found in the changelog below.
 
 ---
@@ -940,15 +941,15 @@
 ## Installation Instructions
 
 The base compiler can be installed as an opam switch with the following commands:
 ```bash
 opam update
 opam switch create 5.2.0
 ```
 
 The source code for the release candidate is also directly available on:
 
 * [GitHub](https://github.com/ocaml/ocaml/archive/5.2.0.tar.gz)
 * [OCaml archives at Inria](https://caml.inria.fr/pub/distrib/ocaml-5.2/ocaml-5.2.0.tar.gz)
 
 ### Fine-Tuned Compiler Configuration
 

data/changelog/releases/ocaml/2024-11-18-ocaml-5.2.1.md

@@ -2,6 +2,7 @@
 title: Release of OCaml 5.2.1
 description: Release of OCaml 5.2.1
 tags: [ocaml]
+versions: ["OCaml 5.2.1"]
 changelog: |
 
   ## Changes Since OCaml 5.2.0

data/changelog/releases/ocaml/2025-01-08-ocaml-5.3.0.md

@@ -2,6 +2,7 @@
 title: Release of OCaml 5.3.0
 description: Release of OCaml 5.3.0
 tags: [ocaml]
+versions: ["OCaml 5.3.0"]
 changelog: |
   (Changes that can break existing programs are marked with a "*")
   ### Restored backend:

data/changelog/releases/ocamlformat/2022-07-18-ocamlformat-0.24.1.md

@@ -1,6 +1,7 @@
 ---
 title: OCamlFormat 0.24.1
 tags: [ocamlformat, platform]
+versions: ["0.24.0", "0.24.1"]
 changelog: |
   ### New features
 

data/changelog/releases/ocamlformat/2023-03-06-ocamlformat-0.25.1.md

@@ -1,6 +1,7 @@
 ---
 title: OCamlFormat 0.25.1
 tags: [ocamlformat, platform]
+versions: ["0.25.0", "0.25.1"]
 changelog: |
   ### Library
 
@@ -171,4 +172,4 @@ The OCamlFormat team
   +         else if Sys.unix then (module Unix)
   +         else (module Fail)
   +        : Unix_socket)
-  ```
+  ```

data/changelog/releases/omp/2020-04-15-omp-1.7.1.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-1.7.1
 tags: [omp]
+versions: ["v1.7.1"]
 changelog: |
   - Fix build with OCaml < 4.08
 

data/changelog/releases/omp/2020-04-20-omp-1.7.2.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-1.7.2
 tags: [omp]
+versions: ["v1.7.2"]
 changelog: |
   - Remove toplevel `Option` module accidentally added in 1.7.0
 ---

data/changelog/releases/omp/2020-05-07-omp-1.7.3.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-1.7.3
 tags: [omp]
+versions: ["v1.7.3"]
 changelog: |
   - Fix magic numbers for the 4.11 ast (#96, @hhugo)
 ---

data/changelog/releases/omp/2020-08-12-omp-2.0.0.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-2.0.0
 tags: [omp]
+versions: ["v2.0.0"]
 changelog: |
   - No longer expose the unwrapped modules (#94, @jonludlam)
   - Remove everything but Ast versions and upgrade/downgrade

data/changelog/releases/omp/2020-10-22-omp-2.1.0.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-2.1.0
 tags: [omp]
+versions: ["v2.1.0"]
 changelog: |
   - Add support for 4.12 (#107, @ceastlund)
 ---

data/changelog/releases/omp/2020-10-23-omp-1.8.0.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-1.8.0
 tags: [omp]
+versions: ["v1.8.0"]
 changelog: |
   Oops, we went looking but didn't find the changelog for this release 🙈
 ---

data/changelog/releases/omp/2021-06-22-omp-2.2.0.md

@@ -1,6 +1,7 @@
 ---
 title: Omp ocaml-migrate-parsetree-2.2.0
 tags: [omp]
+versions: ["v2.2.0"]
 changelog: |
   - Add support for 4.13 (#114, @kit-ty-kate)
 ---

data/changelog/releases/opam-publish/2018-09-19-opam-publish-2.0.0.md

@@ -1,6 +1,7 @@
 ---
 title: Opam-publish 2.0.0
 tags: [opam-publish, platform]
+versions: ["2.0.0", "2.0.0: Merge pull request #66 from rjbou/push-on-master"]
 changelog: |
   * Switch default branch from 2.0.0 to master
   * Minor fix

data/changelog/releases/opam/2024-05-22-opam-2-1-6.md

@@ -1,6 +1,7 @@
 ---
 title: opam 2.1.5
 authors: [ "Raja Boujbel" ]
+versions: ["2.1.6"]
 description: "Release of opam 2.1.5"
 tags: [opam, platform]
 changelog: |

data/changelog/releases/opam/2024-07-01-opam-2-2-0.md

@@ -5,6 +5,7 @@ authors: [
   "Kate Deplaix",
   "David Allsopp",
 ]
+versions: ["2.2.0"]
 description: "Release of opam 2.2.0"
 tags: [opam, platform]
 ---

data/changelog/releases/opam/2024-08-22-opam-2-2-1.md

@@ -5,6 +5,7 @@ authors: [
   "Kate Deplaix",
   "David Allsopp",
 ]
+versions: ["2.2.1"]
 description: "opam 2.2.1 release"
 tags: [opam, platform]
 ---

data/changelog/releases/opam/2024-11-13-opam-2-3-0.md

@@ -5,6 +5,7 @@ authors: [
   "Kate Deplaix",
   "David Allsopp",
 ]
+versions: ["2.3.0"]
 description: "Release of opam 2.3.0"
 tags: [opam, platform]
 ---

src/ocamlorg_data/data_intf.ml

@@ -98,6 +98,8 @@ module Changelog = struct
     body : string;
     authors : string list;
     contributors : string list;
+    project_name : string;
+    versions : string list;
   }
 
   type post = {

tool/ood-gen/bin/scrape.ml

@@ -1,7 +1,12 @@
 open Cmdliner
 open Ood_gen
 
-let term_scrapers = [ ("planet", Blog.Scraper.scrape); ("video", Video.scrape) ]
+let term_scrapers =
+  [
+    ("planet", Blog.Scraper.scrape);
+    ("video", Video.scrape);
+    ("changelog", Changelog.Scraper.scrape);
+  ]
 
 let cmds =
   Cmd.group (Cmd.info "ood-scrape")

tool/ood-gen/lib/changelog.ml

@@ -41,8 +41,37 @@ type t = [%import: Data_intf.Changelog.t] [@@deriving of_yaml, show]
    a pull request to mirror a release announcement that likely already happened
    on discuss.ocaml.org. *)
 
-let re_date_slug =
+(** A scraper is provided to check whether changelog entries are missing. Run it
+    like this:
+    {v
+     dune exec -- tool/ood-gen/bin/scrape.exe changelog
+    v}
+    The list below describes how to query the latest releases. *)
+let projects_release_feeds =
+  [
+    ("ocamlformat", `Github "https://github.com/ocaml-ppx/ocamlformat");
+    ("dune", `Github "https://github.com/ocaml/dune");
+    ("dune-release", `Github "https://github.com/tarides/dune-release");
+    ("mdx", `Github "https://github.com/realworldocaml/mdx");
+    ("merlin", `Github "https://github.com/ocaml/merlin");
+    ("ocaml", `Github "https://github.com/ocaml/ocaml");
+    ("ocaml-lsp", `Github "https://github.com/ocaml/ocaml-lsp");
+    ("ocp-indent", `Github "https://github.com/OCamlPro/ocp-indent");
+    ("odoc", `Github "https://github.com/ocaml/odoc");
+    ("opam", `Github "https://github.com/ocaml/opam/");
+    ("opam-publish", `Github "https://github.com/ocaml-opam/opam-publish");
+    ("ppxlib", `Github "https://github.com/ocaml-ppx/ppxlib");
+    ("utop", `Github "https://github.com/ocaml-community/utop");
+    ("omp", `Github "https://github.com/ocaml-ppx/ocaml-migrate-parsetree");
+  ]
+
+let re_slug =
   let open Re in
+  let re_project_name =
+    let w = rep1 alpha in
+    seq [ w; rep (seq [ char '-'; w ]) ]
+  in
+  let re_version_string = seq [ digit; rep1 any ] in
   compile
     (seq
        [
@@ -56,17 +85,21 @@ let re_date_slug =
              group (rep1 digit);
            ];
          char '-';
+         opt
+           (seq
+              [ group re_project_name; set "-."; group re_version_string; eos ]);
        ])
 
-let parse_date_from_slug s =
-  match Re.exec_opt re_date_slug s with
+let parse_slug s =
+  match Re.exec_opt re_slug s with
   | None -> None
   | Some g ->
       let int n = Re.Group.get g n |> int_of_string in
       let year = int 1 in
       let month = int 2 in
       let day = int 3 in
-      Some (Printf.sprintf "%04d-%02d-%02d" year month day)
+      let version = Re.Group.get_opt g 5 in
+      Some (Printf.sprintf "%04d-%02d-%02d" year month day, version)
 
 module Releases = struct
   type release_metadata = {
@@ -76,18 +109,21 @@ module Releases = struct
     contributors : string list option;
     description : string option;
     changelog : string option;
+    versions : string list option;
   }
   [@@deriving
     of_yaml,
       stable_record ~version:release ~remove:[ changelog; description ]
-        ~modify:[ authors; contributors ]
-        ~add:[ slug; changelog_html; body_html; body; date ]]
+        ~modify:[ authors; contributors; versions ]
+        ~add:[ slug; changelog_html; body_html; body; date; project_name ]]
 
   let of_release_metadata m =
     release_metadata_to_release m ~modify_authors:(Option.value ~default:[])
       ~modify_contributors:(Option.value ~default:[])
+      ~modify_versions:(Option.value ~default:[])
 
   let decode (fname, (head, body)) =
+    let project_name = Filename.basename (Filename.dirname fname) in
     let slug = Filename.basename (Filename.remove_extension fname) in
     let metadata =
       release_metadata_of_yaml head |> Result.map_error (Utils.where fname)
@@ -109,16 +145,21 @@ module Releases = struct
                 |> Hilite.Md.transform
                 |> Cmarkit_html.of_doc ~safe:false)
         in
-        let date =
-          match parse_date_from_slug slug with
+        let date, slug_version =
+          match parse_slug slug with
           | Some x -> x
           | None ->
               failwith
-                "date is not present in metadata and could not be parsed from \
-                 slug"
+                ("date is not present in metadata and could not be parsed from \
+                  slug: " ^ slug)
+        in
+        let metadata =
+          match (metadata.versions, slug_version) with
+          | None, Some v -> { metadata with versions = Some [ v ] }
+          | _ -> metadata
         in
         of_release_metadata ~slug ~changelog_html ~body ~body_html ~date
-          metadata)
+          ~project_name metadata)
       metadata
 
   let all () =
@@ -154,8 +195,8 @@ module Posts = struct
     Result.map
       (fun metadata ->
         let date =
-          match parse_date_from_slug slug with
-          | Some x -> x
+          match parse_slug slug with
+          | Some (date, _) -> date
           | None ->
               failwith
                 "date is not present in metadata and could not be parsed from \
@@ -233,3 +274,54 @@ include Data_intf.Changelog
 let all = %a
 |ocaml}
     (Fmt.Dump.list pp) (all ())
+
+module Scraper = struct
+  module SMap = Map.Make (String)
+  module SSet = Set.Make (String)
+
+  let warning_count = ref 0
+
+  let warn fmt =
+    let flush out =
+      Printf.fprintf out "\n%!";
+      incr warning_count
+    in
+    Printf.kfprintf flush stderr fmt
+
+  let fetch_github repo =
+    [ River.fetch { River.name = repo; url = repo ^ "/releases.atom" } ]
+    |> River.posts
+    |> List.map (fun post -> River.title post)
+
+  let group_releases_by_project all =
+    List.fold_left
+      (fun acc t ->
+        List.fold_left
+          (fun acc v -> SMap.add_to_list t.project_name v acc)
+          acc t.versions)
+      SMap.empty all
+
+  let check_if_uptodate project known_versions =
+    let known_versions = SSet.of_list known_versions in
+    let check scraped_versions =
+      List.iter
+        (fun v ->
+          if not (SSet.mem v known_versions) then
+            warn "No changelog entry for %S version %S\n%!" project v)
+        scraped_versions
+    in
+    match List.assoc_opt project projects_release_feeds with
+    | Some (`Github repo) -> check (fetch_github repo)
+    | None ->
+        warn
+          "Don't know how to lookup project %S. Please update \
+           'tool/ood-gen/lib/changelog.ml'\n\
+           %!"
+          project
+
+  (** This does not generate any file. Instead, it exits with an error if a
+      changelog entry is missing. *)
+  let scrape () =
+    Releases.all () |> group_releases_by_project |> SMap.iter check_if_uptodate;
+    if !warning_count > 0 then exit 1
+end

tool/ood-gen/test/test_ood_gen.ml

@@ -3,14 +3,23 @@ let test_parse_date_from_slug =
     ( name,
       `Quick,
       fun () ->
-        let got = Ood_gen.Changelog.parse_date_from_slug s in
-        Alcotest.check (Alcotest.option Alcotest.string) __LOC__ expected got )
+        let got = Ood_gen.Changelog.parse_slug s in
+        Alcotest.(check (option (pair string (option string))))
+          __LOC__ expected got )
   in
   [
-    test ~name:"ok" "2020-03-02-something.md" ~expected:(Some "2020-03-02");
+    test ~name:"ok" "2020-03-02-something.md"
+      ~expected:(Some ("2020-03-02", None));
     test ~name:"no date" "something.md" ~expected:None;
     test ~name:"day not padded correctly" "2021-1-2-title.md"
-      ~expected:(Some "2021-01-02");
+      ~expected:(Some ("2021-01-02", None));
+    test ~name:"ok with project-version" "2025-01-31-project-1.2.3"
+      ~expected:(Some ("2025-01-31", Some "1.2.3"));
+    test ~name:"ok with project.version" "2025-01-31-project.1.2.3"
+      ~expected:(Some ("2025-01-31", Some "1.2.3"));
+    test ~name:"ok with project.version-suffix"
+      "2025-01-31-project.1.2.3-something"
+      ~expected:(Some ("2025-01-31", Some "1.2.3-something"));
   ]
 
 let tests = [ ("parse_date_from_slug", test_parse_date_from_slug) ]