diff --git a/deps.edn b/deps.edn index 4a97e85..c8b71e5 100644 --- a/deps.edn +++ b/deps.edn @@ -1,5 +1,5 @@ {:paths ["src" "resources"] - :deps {org.clojure/clojure {:mvn/version "1.12.0-alpha3"} + :deps {org.clojure/clojure {:mvn/version "1.12.0"} net.cgrand/xforms {:mvn/version "0.19.5"} com.phronemophobic/clong {:mvn/version "1.4.2"} org.clojure/data.priority-map {:mvn/version "1.1.0"} diff --git a/src/com/phronemophobic/clj_media.clj b/src/com/phronemophobic/clj_media.clj index e685f32..eea56a5 100644 --- a/src/com/phronemophobic/clj_media.clj +++ b/src/com/phronemophobic/clj_media.clj @@ -14,6 +14,7 @@ [com.phronemophobic.clj-media.impl.raw :as raw] [com.phronemophobic.clj-media.impl.av :as av])) +(set! *warn-on-reflection* true) (def sample-formats "All possible sample formats." @@ -174,7 +175,7 @@ (transduce (comp (map mm/byte-buffer) - (map (fn [buf] + (map (fn [^java.nio.ByteBuffer buf] (let [bytes (byte-array (.capacity buf))] (.get buf bytes) bytes)))) diff --git a/src/com/phronemophobic/clj_media/impl/audio.clj b/src/com/phronemophobic/clj_media/impl/audio.clj index 5ba5cf5..0fdb94c 100644 --- a/src/com/phronemophobic/clj_media/impl/audio.clj +++ b/src/com/phronemophobic/clj_media/impl/audio.clj @@ -81,7 +81,7 @@ (def format-ctx (avformat_alloc_context)) (def format-ctx* (doto (PointerByReference.) - (.setValue (.getPointer format-ctx)))) + (.setValue (Structure/.getPointer format-ctx)))) (avformat_open_input format-ctx* fname @@ -204,7 +204,7 @@ (.drain source-data-line) (.close source-data-line) read-bytes) - ([read-bytes buf] + ([read-bytes ^byte/1 buf] (+ read-bytes (.write source-data-line buf 0 (alength buf)))))))) @@ -217,7 +217,7 @@ (:nb_samples frame ) (:channels frame )) buf (-> (nth (:data frame ) 0) - (.getPointer ) + Structure/.getPointer (.getByteArray 0 buf-size))] buf))))) @@ -225,11 +225,11 @@ (defn resample2 [input-format output-format] (let [resample-ctx* (PointerByReference. Pointer/NULL) err (swr_alloc_set_opts2 resample-ctx* - (.getPointer + (Structure/.getPointer (:ch-layout output-format)) (:sample-format output-format) (:sample-rate output-format) - (.getPointer + (Structure/.getPointer (:ch-layout input-format)) (:sample-format input-format) (:sample-rate input-format) @@ -269,8 +269,11 @@ output-format ch-layout (AVChannelLayout.)] ;; create out own copy since original copy may change :( - (av_channel_layout_copy (.getPointer ch-layout) - (.getPointer (:ch-layout output-format))) + (av_channel_layout_copy (Structure/.getPointer ch-layout) + (Structure/.getPointer (:ch-layout output-format))) + (comment + (supers AVChannelLayout) + ) (fn [] (let [frame (doto (av/new-frame) @@ -281,8 +284,8 @@ (.writeField "sample_rate" sample-rate))] (assert (zero? (av_channel_layout_copy - (.getPointer (:ch_layout frame)) - (.getPointer ch-layout)))) + (Structure/.getPointer (:ch_layout frame)) + (Structure/.getPointer ch-layout)))) (assert (>= (av_frame_get_buffer frame 0) 0)) @@ -297,7 +300,7 @@ output-frame* (volatile! (new-output-frame))] - (assert (<= num-output-channels (alength (:data (AVFrame.))))) + (assert (<= num-output-channels (alength ^byte/1 (:data (AVFrame.))))) (fn [rf] (fn ([] (rf)) @@ -313,9 +316,9 @@ data-ptr (into-array Pointer (eduction - (map (fn [p] + (map (fn [^AVChannelLayout p] (when p - (.share (.getPointer p) + (.share (Structure/.getPointer p) (* sample-offset-multiplier current-samples))))) (:data output-frame))) @@ -349,7 +352,7 @@ (eduction (map (fn [p] (when p - (.share (.getPointer p) + (.share (Structure/.getPointer p) (* sample-offset-multiplier current-samples))))) (:data output-frame))) @@ -384,7 +387,7 @@ (let [buf-size (first (:linesize frame)) buf (-> (:data frame) (nth 0) - (.getPointer) + (Structure/.getPointer) (.getByteBuffer 0 buf-size))] buf)) diff --git a/src/com/phronemophobic/clj_media/impl/av.clj b/src/com/phronemophobic/clj_media/impl/av.clj index 736801f..29416f4 100644 --- a/src/com/phronemophobic/clj_media/impl/av.clj +++ b/src/com/phronemophobic/clj_media/impl/av.clj @@ -22,16 +22,18 @@ com.sun.jna.Structure) (:gen-class)) +(set! *warn-on-reflection* true) + (raw/import-structs!) -(def cleaner (Cleaner/create)) +(def ^Cleaner cleaner (Cleaner/create)) (defonce handles (atom #{})) (defn ref! [o] (swap! handles conj o) o) -(defn ->avrational [num den] +(defn ->avrational ^AVRational [num den] (doto (AVRational.) (.writeField "num" (int num)) (.writeField "den" (int den)))) @@ -40,13 +42,13 @@ (defn error->str [err] (let [buf (byte-array 255)] (av_strerror err buf (alength buf)) - (let [s (String. buf 0 (transduce - (take-while #(not (zero? %))) - (completing - (fn [cnt _] - (inc cnt))) - 0 - buf))] + (let [s (String. buf 0 (long (transduce + (take-while #(not (zero? %))) + (completing + (fn [cnt _] + (inc cnt))) + 0 + buf)))] s))) @@ -60,8 +62,8 @@ -(defn next-packet [ctx] - (let [packet (av_packet_alloc) +(defn next-packet [^AVFormatContext ctx] + (let [^AVPacket packet (av_packet_alloc) ptr (Pointer/nativeValue (.getPointer packet))] (.register cleaner packet (fn [] @@ -94,10 +96,10 @@ (nativeType [_] Pointer) (toNative [_] - (.toNative frame))) + (com.sun.jna.NativeMapped/.toNative frame))) -(defn new-frame [] - (let [frame (av_frame_alloc) +(defn new-frame ^AVFrame [] + (let [^AVFrame frame (av_frame_alloc) ptr (Pointer/nativeValue (.getPointer frame))] (.register cleaner frame (fn [] @@ -111,7 +113,7 @@ frame)) (defn new-packet[] - (let [packet (av_packet_alloc) + (let [^AVPacket packet (av_packet_alloc) ptr (Pointer/nativeValue (.getPointer packet))] (.register cleaner packet (fn [] @@ -130,7 +132,7 @@ (fn ([] (rf)) ([result] (rf result)) - ([result format-context] + ([result ^Structure format-context] (loop [result result] (let [packet (new-packet) err (av_read_frame (.getPointer format-context) packet) @@ -307,14 +309,15 @@ result)] result)))) -(defn open-context [fname] +(defn open-context + ^AVFormatContext [fname] (let [format-ctx (avformat_alloc_context) _ (when (nil? format-ctx) (throw (ex-info "Error allocating format context." {:filename fname}))) format-ctx* (doto (PointerByReference.) - (.setValue (.getPointer format-ctx))) + (.setValue (Structure/.getPointer format-ctx))) _ (.register cleaner format-ctx (fn [] (avformat_free_context (.getValue format-ctx*)))) @@ -354,7 +357,7 @@ AVMEDIA_TYPE_AUDIO (audio-codec-context-format codec-context) AVMEDIA_TYPE_VIDEO (video-codec-context-format codec-context))) -(defn add-stream [output-format-context encoder-context] +(defn add-stream [output-format-context ^AVCodecContext encoder-context] (let [output-format (:oformat output-format-context) output-format+ (Structure/newInstance AVOutputFormatByReference output-format) @@ -389,13 +392,14 @@ (throw (Exception. "Could not initialize stream params"))) stream)) -(defn video-encoder-context [format] +(defn video-encoder-context ^AVCodecContext [format] (let [codec-id (-> format :codec :id) output-codec (avcodec_find_encoder codec-id) _ (when (nil? output-codec) (throw (Exception. "could not find encoder"))) + ^AVCodecContext encoder-context (avcodec_alloc_context3 output-codec) _ (when (nil? encoder-context) (throw (Exception. "Could not create encoder"))) @@ -437,12 +441,13 @@ (.writeField "bit_rate" bit-rate)))] encoder-context)) -(defn audio-encoder-context [format] +(defn audio-encoder-context ^AVCodecContext [format] (let [codec-id (-> format :codec :id) output-codec (avcodec_find_encoder codec-id) _ (when (nil? output-codec) (throw (Exception. "could not find encoder"))) + ^AVCodecContext encoder-context (avcodec_alloc_context3 output-codec) _ (when (nil? encoder-context) (throw (Exception. "Could not create encoder"))) @@ -451,11 +456,12 @@ sample-fmt (:sample-format format) sample-rate (:sample-rate format) + ^AVChannelLayout ch-layout (:ch-layout format) _ (assert (zero? (av_channel_layout_copy - (.getPointer (:ch_layout encoder-context)) + (AVChannelLayout/.getPointer (:ch_layout encoder-context)) (.getPointer ch-layout)))) _ (doto encoder-context @@ -467,12 +473,12 @@ encoder-context)) -(defn encoder-context [format] +(defn encoder-context ^AVCodecContext [format] (case (:media-type format) :media-type/video (video-encoder-context format) :media-type/audio (audio-encoder-context format))) -(defn find-decoder-context [media-type format-context] +(defn find-decoder-context ^AVCodecContext [media-type ^AVFormatContext format-context] (let [err (avformat_find_stream_info format-context nil) _ (when (not (zero? err)) (throw (ex-info "Could not find stream info." @@ -490,7 +496,7 @@ {:error-code best-stream :media-type media-type}))) num-streams (.readField format-context "nb_streams") - streams (.getPointerArray + streams (Pointer/.getPointerArray (.readField format-context "streams") 0 num-streams) stream (aget streams best-stream) @@ -498,11 +504,13 @@ stream) codec-parameters (.readField stream+ "codecpar") - codec-id (.readField codec-parameters "codec_id" ) + codec-id (Structure/.readField codec-parameters "codec_id" ) + ^AVCodec decoder (avcodec_find_decoder codec-id) _ (when (nil? decoder) (throw (ex-info "Could not find decoder" {:codec-id codec-id}))) + ^AVCodecContext decoder-context (avcodec_alloc_context3 (.getPointer decoder)) _ (when (nil? decoder-context) @@ -593,7 +601,7 @@ (throw (ex-info "Could not find stream info." {:error-code err}))) num-streams (:nb_streams format-context) - streams (.getPointerArray + streams (Pointer/.getPointerArray (.readField format-context "streams") 0 num-streams) @@ -627,7 +635,7 @@ {:streams streams-info})) -(defn make-frame [{:keys [bytes +(defn make-frame [{:keys [^byte/1 bytes format time-base key-frame? @@ -682,8 +690,8 @@ (.writeField "sample_rate" sample-rate)) (assert (zero? (raw/av_channel_layout_copy - (.getPointer (:ch_layout frame)) - (.getPointer ch-layout)))) + (Structure/.getPointer (:ch_layout frame)) + (Structure/.getPointer ch-layout)))) (assert (>= (raw/av_frame_get_buffer frame 0) 0)) @@ -691,13 +699,13 @@ ;; since linesize is sometimes set for a particular alignment ;; I think line size is set by raw/av_frame_get_buffer (when (> (alength bytes) - (aget (:linesize frame) 0)) + (aget ^ints (:linesize frame) 0)) (throw (ex-info "Bytes are the wrong length for sample format." {:frame m :bytes bytes :actual-size (alength bytes) - :expected-length (aget (:linesize frame) 0)}))) - (.write (.getPointer (aget (:data frame) 0)) 0 bytes 0 (alength bytes))) + :expected-length (aget ^ints (:linesize frame) 0)}))) + (.write (Structure/.getPointer (aget ^objects (:data frame) 0)) 0 bytes 0 (alength bytes))) :media-type/video (let [{:keys [pixel-format @@ -711,14 +719,14 @@ (doto frame (.writeField "linesize" (doto (int-array 8) - (aset 0 line-size)))) + (aset 0 (int line-size))))) (throw (ex-info ":line-size must be set when creating video frames." {:frame m}))) (assert (>= (raw/av_frame_get_buffer frame 0) 0)) - (.write (.getPointer (aget (:data frame) 0)) 0 bytes 0 (alength bytes))) + (.write (Structure/.getPointer (aget ^objects (:data frame) 0)) 0 bytes 0 (alength bytes))) nil (throw (ex-info "frame requires `:media-type` to be set." {:frame m}))) diff --git a/src/com/phronemophobic/clj_media/impl/datafy.clj b/src/com/phronemophobic/clj_media/impl/datafy.clj index 0e3ba17..52bcf38 100644 --- a/src/com/phronemophobic/clj_media/impl/datafy.clj +++ b/src/com/phronemophobic/clj_media/impl/datafy.clj @@ -15,6 +15,8 @@ com.sun.jna.Pointer com.sun.jna.ptr.PointerByReference)) +(set! *warn-on-reflection* true) + (raw/import-structs!) (defn ch-layout->str [ch-layout] @@ -26,7 +28,7 @@ :err err}))) (String. (.getByteArray buf 0 err) "ascii"))) -(defn str->ch-layout [s] +(defn ^AVChannelLayoutByReference str->ch-layout [s] (assert s "Invalid ch-layout.") (let [ch-layout (AVChannelLayoutByReference.) err (av_channel_layout_from_string ch-layout s)] @@ -35,7 +37,7 @@ {:channel-layout-str s}))) ch-layout)) -(defn pointer-seq [p size terminal] +(defn pointer-seq [^Pointer p ^long size terminal] (loop [offset 0 results []] (let [x (case size @@ -48,7 +50,7 @@ (def ^:private avrational-size (.size (AVRational.))) (defn avrational-seq [p] - (loop [p p + (loop [^Pointer p p results []] (let [ratio (Structure/newInstance AVRationalByReference p)] (if (and (zero? (:num ratio)) @@ -60,7 +62,7 @@ (def ^:private avchannellayout-size (.size (AVChannelLayout.))) (defn avchannellayout-seq [p] - (loop [p p + (loop [^Pointer p p results []] (let [bs (.getByteArray p 0 avchannellayout-size)] (if (every? zero? bs) @@ -85,7 +87,7 @@ (defmulti read-bytes (fn [type bs] type)) (defmethod read-bytes :avoption-type/int64 - [_ bs] + [_ ^byte/1 bs] (let [_ (assert (= 8 (alength bs))) bs (if (= (ByteOrder/nativeOrder) ByteOrder/LITTLE_ENDIAN) @@ -106,7 +108,7 @@ (not (zero? num)))) (defmethod read-bytes :avoption-type/uint64 - [_ bs] + [_ ^byte/1 bs] (let [_ (assert (= 8 (alength bs))) bs (if (= (ByteOrder/nativeOrder) ByteOrder/LITTLE_ENDIAN) @@ -308,7 +310,7 @@ :name (ch-layout->str p) :nb-channels (:nb_channels p)} (when (= :channel-order/native) - (let [bs (:u p) + (let [^byte/1 bs (:u p) _ (assert (= 8 (alength bs))) bs (if (= (ByteOrder/nativeOrder) ByteOrder/LITTLE_ENDIAN) diff --git a/src/com/phronemophobic/clj_media/impl/java2d.clj b/src/com/phronemophobic/clj_media/impl/java2d.clj index 4a25a80..a5c3f5e 100644 --- a/src/com/phronemophobic/clj_media/impl/java2d.clj +++ b/src/com/phronemophobic/clj_media/impl/java2d.clj @@ -16,10 +16,13 @@ [tech.v3.libs.buffered-image :as bufimg] ) (:import com.sun.jna.Pointer + com.sun.jna.Structure java.awt.Graphics2D java.awt.Color java.awt.RenderingHints)) +(set! *warn-on-reflection* true) + (raw/import-structs!) (defn play-video [fname] @@ -41,6 +44,7 @@ repaint (::java2d/repaint window-info) start-time (System/currentTimeMillis) + ^AVRational time-base (.readField decoder-context "time_base") num (.readField time-base "num") den (.readField time-base "den") @@ -54,6 +58,7 @@ (try (loop [t 0] (let [start-frame-time (System/currentTimeMillis) + ^AVFrame frame (loop [] (let [packet (av/next-packet input-context)] (when packet @@ -63,7 +68,7 @@ (recur))))))] (when frame - (let [best-effort-timestamp (.readField frame "best_effort_timestamp") + (let [best-effort-timestamp (long (.readField frame "best_effort_timestamp")) sleep-ms (- (* 1000 fps best-effort-timestamp) (- (System/currentTimeMillis) start-time ))] (when (pos? sleep-ms) @@ -112,33 +117,49 @@ (java2d/draw elem)) img))) -(defn raster-data [img] +(defn raster-data [^java.awt.image.BufferedImage img] (let [raster (.getData img) buffer (.getDataBuffer raster) - buf (.getData buffer)] + buf (cond + (instance? java.awt.image.DataBufferByte buffer) (.getData ^java.awt.image.DataBufferByte buffer) + (instance? java.awt.image.DataBufferDouble buffer) (.getData ^java.awt.image.DataBufferDouble buffer) + (instance? java.awt.image.DataBufferFloat buffer) (.getData ^java.awt.image.DataBufferFloat buffer) + (instance? java.awt.image.DataBufferInt buffer) (.getData ^java.awt.image.DataBufferInt buffer) + (instance? java.awt.image.DataBufferShort buffer) (.getData ^java.awt.image.DataBufferShort buffer) + (instance? java.awt.image.DataBufferUShort buffer) (.getData ^java.awt.image.DataBufferUShort buffer) + :else (throw (ex-info (str "Unknown class: " (class buffer)) {})))] buf)) -(defn img->frame [img] +(defn img->frame [^java.awt.image.BufferedImage img] (let [w (.getWidth img) h (.getHeight img) pix-fmt AV_PIX_FMT_RGB24 - frame (doto (av_frame_alloc) - (.writeField "width" w) - (.writeField "height" h) - (.writeField "format" pix-fmt)) - + ^AVFrame frame (av_frame_alloc) + _ (doto frame + (.writeField "width" w) + (.writeField "height" h) + (.writeField "format" pix-fmt)) err av_frame_get_buffer] (assert (zero? (av_frame_get_buffer frame 0))) (assert (zero? (av_frame_make_writable frame))) (let [data-ptr (-> (.readField frame "data") first - (.getPointer)) + (Structure/.getPointer)) buf (raster-data img) bytes-per-pixel 3 source-linesize (* bytes-per-pixel w) dest-linesize (first (.readField frame "linesize"))] - (doseq [y (range h)] - (.write data-ptr (* y dest-linesize) buf (* y source-linesize) source-linesize))) + (doseq [y (range h) + :let [offset (long (* y dest-linesize)) + index (long (* y source-linesize))]] + (cond + (instance? byte/1 buf) (.write data-ptr offset ^byte/1 buf index source-linesize) + (instance? char/1 buf) (.write data-ptr offset ^char/1 buf index source-linesize) + (instance? double/1 buf) (.write data-ptr offset ^double/1 buf index source-linesize) + (instance? float/1 buf) (.write data-ptr offset ^float/1 buf index source-linesize) + (instance? int/1 buf) (.write data-ptr offset ^int/1 buf index source-linesize) + (instance? long/1 buf) (.write data-ptr offset ^long/1 buf index source-linesize) + (instance? short/1 buf) (.write data-ptr offset ^short/1 buf index source-linesize)))) frame)) diff --git a/src/com/phronemophobic/clj_media/impl/raw.clj b/src/com/phronemophobic/clj_media/impl/raw.clj index ccca7db..3caf2fa 100644 --- a/src/com/phronemophobic/clj_media/impl/raw.clj +++ b/src/com/phronemophobic/clj_media/impl/raw.clj @@ -54,8 +54,8 @@ (Class/forName (name classname)) (catch ClassNotFoundException e nil))] - (when-let [constructor (.getConstructor cls - (into-array Class []))] + (when-let [constructor (Class/.getConstructor cls + (into-array Class []))] (.newInstance constructor (to-array []))))) (run! try-load @@ -109,12 +109,13 @@ @(requiring-resolve 'com.phronemophobic.clong.clang/default-arguments) header-files (->> lib-names - (map #(str "lib" % "/" % ".h")) - (map #(io/file "../FFmpeg/" %)) - (map #(.getCanonicalPath %))) + (map #(->> (str "lib" % "/" % ".h") + (io/file "../FFmpeg/" ) + .getCanonicalPath))) header-files (into header-files - (comp (map #(io/file "../FFmpeg/" %)) - (map #(.getCanonicalPath %))) + (map #(->> % + (io/file "../FFmpeg/") + .getCanonicalPath)) ["libavfilter/buffersink.h" "libavfilter/buffersrc.h"]) @@ -128,8 +129,7 @@ ["-include" h])) (rest header-files)) clang-args (into clang-args - (comp (map io/file) - (map #(.getParent %)) + (comp (map #(-> % io/file .getParent)) (distinct) (map (fn [folder] (str "-I" folder))))