Add pcm s16 format

CHANGES.md

@@ -2,6 +2,8 @@
 
 New:
 
+- Added support for less memory hungry audio formats, namely
+  `pcm_s16` and `pcm_f32` (#3008)
 - Added support for native osc library (#2426, #2480).
 - SRT: added support for passphrase, pbkeylen, streamid,
   added native type for srt sockets with methods, moved stats

src/core/decoder/external_decoder.ml

@@ -91,13 +91,7 @@ let create_stream process input =
   Gc.finalise close ret;
   ret
 
-let audio_n n =
-  Content.(
-    Audio.lift_params
-      {
-        Content.channel_layout =
-          lazy (Audio_converter.Channel_layout.layout_of_channels n);
-      })
+let audio_n = Frame_base.format_of_channels ~pcm_kind:Content.Audio.kind
 
 let test_ctype f filename =
   (* 0 = file rejected,

src/core/decoder/ffmpeg_decoder.ml

@@ -598,16 +598,10 @@ let get_type ~ctype ~url container =
                    Ffmpeg_raw_content.(
                      Audio.lift_params (AudioSpecs.mk_params p)));
               Frame.Fields.add field format content_type
-          | p, Some _ ->
+          | p, Some format ->
               Frame.Fields.add field
-                Content.(
-                  Audio.lift_params
-                    {
-                      Content.channel_layout =
-                        lazy
-                          (Audio_converter.Channel_layout.layout_of_channels
-                             (Avcodec.Audio.get_nb_channels p));
-                    })
+                (Frame_base.format_of_channels ~pcm_kind:(Content.kind format)
+                   (Avcodec.Audio.get_nb_channels p))
                 content_type
           | _ -> content_type)
       Frame.Fields.empty audio_streams
@@ -790,7 +784,30 @@ let mk_streams ~ctype ~decode_first_metadata container =
                     ( stream,
                       check_metadata stream
                         (Ffmpeg_internal_decoder.mk_audio_decoder ~channels
-                           ~stream ~field params) ))
+                           ~stream ~field ~pcm_kind:Content.Audio.kind params)
+                    ))
+                  streams,
+                pos + 1 )
+          | Some format when Content_pcm_s16.is_format format ->
+              let channels = Content_pcm_s16.channels_of_format format in
+              ( Streams.add idx
+                  (`Audio_frame
+                    ( stream,
+                      check_metadata stream
+                        (Ffmpeg_internal_decoder.mk_audio_decoder ~channels
+                           ~stream ~field ~pcm_kind:Content_pcm_s16.kind params)
+                    ))
+                  streams,
+                pos + 1 )
+          | Some format when Content_pcm_f32.is_format format ->
+              let channels = Content_pcm_f32.channels_of_format format in
+              ( Streams.add idx
+                  (`Audio_frame
+                    ( stream,
+                      check_metadata stream
+                        (Ffmpeg_internal_decoder.mk_audio_decoder ~channels
+                           ~stream ~field ~pcm_kind:Content_pcm_f32.kind params)
+                    ))
                   streams,
                 pos + 1 )
           | _ -> (streams, pos + 1))

src/core/decoder/ffmpeg_internal_decoder.ml

@@ -26,11 +26,60 @@ open Mm
 
 let log = Log.make ["decoder"; "ffmpeg"; "internal"]
 
+module type Converter_type = sig
+  type t
+
+  module Content : sig
+    type data
+
+    val lift_data : ?offset:int -> ?length:int -> data -> Content_base.data
+  end
+
+  val create :
+    ?options:Swresample.options list ->
+    Avutil.Channel_layout.t ->
+    ?in_sample_format:Avutil.Sample_format.t ->
+    int ->
+    Avutil.Channel_layout.t ->
+    ?out_sample_format:Avutil.Sample_format.t ->
+    int ->
+    t
+
+  val convert :
+    ?offset:int -> ?length:int -> t -> Swresample.Frame.t -> Content.data
+end
+
 module ConverterInput = Swresample.Make (Swresample.Frame)
-module Converter = ConverterInput (Swresample.PlanarFloatArray)
+
+module Converter = struct
+  module Content = Content_audio
+  include ConverterInput (Swresample.PlanarFloatArray)
+end
+
+module Converter_pcm_s16 = struct
+  module Content = Content_pcm_s16
+  include ConverterInput (Swresample.S16PlanarBigArray)
+end
+
+module Converter_pcm_f32 = struct
+  module Content = Content_pcm_f32
+  include ConverterInput (Swresample.FltPlanarBigArray)
+end
+
 module Scaler = Swscale.Make (Swscale.Frame) (Swscale.BigArray)
 
-let mk_audio_decoder ~channels ~stream ~field codec =
+let mk_audio_decoder ~channels ~stream ~field ~pcm_kind codec =
+  let converter =
+    match pcm_kind with
+      | _ when Content_audio.is_kind pcm_kind ->
+          (module Converter : Converter_type)
+      | _ when Content_pcm_s16.is_kind pcm_kind ->
+          (module Converter_pcm_s16 : Converter_type)
+      | _ when Content_pcm_f32.is_kind pcm_kind ->
+          (module Converter_pcm_f32 : Converter_type)
+      | _ -> raise Content_base.Invalid
+  in
+  let module Converter = (val converter : Converter_type) in
   Ffmpeg_decoder_common.set_audio_stream_decoder stream;
   let in_sample_rate = ref (Avcodec.Audio.get_sample_rate codec) in
   let in_channel_layout = ref (Avcodec.Audio.get_channel_layout codec) in
@@ -59,7 +108,8 @@ let mk_audio_decoder ~channels ~stream ~field codec =
       in_sample_format := frame_in_sample_format;
       converter := mk_converter ());
     let content = Converter.convert !converter frame in
-    buffer.Decoder.put_pcm ~field ~samplerate:target_sample_rate content;
+    Generator.put buffer.Decoder.generator field
+      (Converter.Content.lift_data content);
     let metadata = Avutil.Frame.metadata frame in
     if metadata <> [] then (
       let m = Hashtbl.create (List.length metadata) in

src/core/decoder/ffmpeg_internal_decoder.mli

@@ -24,6 +24,7 @@ val mk_audio_decoder :
   channels:int ->
   stream:(Avutil.input, Avutil.audio, [ `Frame ]) Av.stream ->
   field:Frame.field ->
+  pcm_kind:Content.kind ->
   Avutil.audio Avcodec.params ->
   buffer:Decoder.buffer ->
   Avutil.audio Avutil.Frame.t ->

src/core/decoder/gstreamer_decoder.ml

@@ -277,14 +277,7 @@ let get_type ~channels filename =
   in
   let audio =
     if audio = 0 then None
-    else
-      Some
-        Content.(
-          Audio.lift_params
-            {
-              Content.channel_layout =
-                lazy (Audio_converter.Channel_layout.layout_of_channels audio);
-            })
+    else Some (Frame_base.format_of_channels ~pcm_kind:Content.Audio.kind audio)
   in
   let video =
     if video = 0 then None else Some Content.(default_format Video.kind)

src/core/decoder/liq_ogg_decoder.ml

@@ -250,13 +250,7 @@ let file_type ~ctype:_ filename =
         if audio = 0 then None
         else
           Some
-            Content.(
-              Audio.lift_params
-                {
-                  Content.channel_layout =
-                    lazy
-                      (Audio_converter.Channel_layout.layout_of_channels audio);
-                })
+            (Frame_base.format_of_channels ~pcm_kind:Content.Audio.kind audio)
       in
       let video =
         if video = 0 then None else Some Content.(default_format Video.kind)

src/core/decoder/wav_aiff_decoder.ml

@@ -151,14 +151,8 @@ let file_type ~ctype:_ filename =
       Some
         (Frame.Fields.make
            ~audio:
-             Content.(
-               Audio.lift_params
-                 {
-                   Content.channel_layout =
-                     lazy
-                       (Audio_converter.Channel_layout.layout_of_channels
-                          channels);
-                 })
+             (Frame_base.format_of_channels ~pcm_kind:Content.Audio.kind
+                channels)
            ()))
 
 let wav_mime_types =

src/core/dune

@@ -68,6 +68,9 @@
   compress_exp
   content
   content_audio
+  content_pcm_base
+  content_pcm_f32
+  content_pcm_s16
   content_base
   content_midi
   content_timed
@@ -231,6 +234,7 @@
   theora_format
   time_warp
   track
+  track_map
   tutils
   type
   typing