Start setting up the audio control system

All of the examples are for switching out elements in the middle of a
pipeline. In this case I am trying to switch out the trailing element.
The element refuses to be removed, and a new one can't be added until
the old audio sink is removed. I think that the audio sink can't be
removed because it is still holding on to data, and I don't know how to
detect the EOS signal as it passes through.

flake.lock

@@ -20,16 +20,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1704732714,
-        "narHash": "sha256-ABqK/HggMYA/jMUXgYyqVAcQ8QjeMyr1jcXfTpSHmps=",
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6723fa4e4f1a30d42a633bef5eb01caeb281adc3",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "type": "indirect"
       }
     },

flake.nix

@@ -2,7 +2,7 @@
   description = "Lumenescent Dreams Tools";
 
   inputs = {
-    nixpkgs.url = "nixpkgs/nixos-23.11";
+    nixpkgs.url = "nixpkgs/nixos-24.05";
     unstable.url = "nixpkgs/nixos-unstable";
     typeshare.url = "github:1Password/typeshare";
   };
@@ -30,6 +30,9 @@
             pkgs.gst_all_1.gst-plugins-good
             pkgs.gst_all_1.gst-plugins-ugly
             pkgs.gst_all_1.gstreamer
+            pkgs.gst_all_1.gstreamer.dev
+            pkgs.gst_all_1.gst-libav
+            pkgs.gst_all_1.gst-vaapi
             pkgs.gtk4
             pkgs.libadwaita
             pkgs.librsvg

gm-dash/server/Cargo.toml

@@ -6,8 +6,10 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-pipewire = "0.8.0"
-serde = { version = "1.0.209", features = ["alloc", "derive"] }
-serde_json = "1.0.127"
-tokio = { version = "1.39.3", features = ["full"] }
-warp = "0.3.7"
+gstreamer   = { version = "0.23.0", features = ["serde", "v1_24"] }
+pipewire    = { version = "0.8.0" }
+serde       = { version = "1.0.209", features = ["alloc", "derive"] }
+serde_json  = { version = "1.0.127" }
+tokio       = { version = "1.39.3", features = ["full"] }
+warp        = { version = "0.3.7" }
+glib        = { version = "0.18" }

gm-dash/server/src/app.rs

@@ -0,0 +1,97 @@
+use crate::audio_control::AudioControl;
+use std::{
+    collections::HashSet,
+    sync::{Arc, RwLock},
+};
+
+struct AppState {
+    device_list: Vec<String>,
+    track_list: Vec<String>,
+    currently_playing: HashSet<String>,
+
+    audio_control: AudioControl,
+}
+
+impl Default for AppState {
+    fn default() -> Self {
+        Self {
+            device_list: vec![],
+            track_list: vec![
+                "/home/savanni/Music/Travis Savoie/RPG Toolkit Volume II/01 - A Day to Rebuild.mp3.mp3".to_owned(),
+                "/home/savanni/Music/Travis Savoie/RPG Toolkit Volume II/02 - Against the Clock.mp3.mp3".to_owned(),
+                "/home/savanni/Music/Travis Savoie/RPG Toolkit Volume II/03 - Alleyway Cutthroat.mp3.mp3".to_owned(),
+                "/home/savanni/Music/Travis Savoie/RPG Toolkit Volume II/04 - Beasts Of Legend.mp3.mp3".to_owned(),
+                "/home/savanni/Music/Travis Savoie/RPG Toolkit Volume II/05 - Books and Spellcrafting.mp3.mp3".to_owned(),
+            ],
+            currently_playing: HashSet::default(),
+
+            audio_control: AudioControl::default(),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct App {
+    internal: Arc<RwLock<AppState>>,
+}
+
+impl App {
+    fn new() -> App {
+        let internal = AppState::default();
+        App {
+            internal: Arc::new(RwLock::new(internal)),
+        }
+    }
+
+    pub fn add_audio(&self, device: String) {
+        let mut st = self.internal.write().unwrap();
+        st.device_list.push(device);
+    }
+
+    pub fn audio_devices(&self) -> Vec<String> {
+        let st = self.internal.read().unwrap();
+        st.device_list.clone()
+    }
+
+    pub fn tracks(&self) -> Vec<String> {
+        let st = self.internal.read().unwrap();
+        st.track_list.clone()
+    }
+
+    pub fn play_pause(&self) -> Result<(), String> {
+        self.internal.write().unwrap().audio_control.play_pause();
+        Ok(())
+    }
+
+    pub fn add_track(&self, track: String) -> Result<(), String> {
+        let mut st = self.internal.write().unwrap();
+        if st.track_list.contains(&track) {
+            st.currently_playing.insert(track.clone());
+        }
+        st.audio_control.add_track(track);
+        Ok(())
+    }
+
+    pub fn stop(&self, track: String) -> Result<(), String> {
+        let mut st = self.internal.write().unwrap();
+        st.currently_playing.remove(&track);
+        Ok(())
+    }
+
+    pub fn stop_all(&self) -> Result<(), String> {
+        let mut st = self.internal.write().unwrap();
+        st.currently_playing = HashSet::new();
+        Ok(())
+    }
+
+    pub fn playing(&self) -> Vec<String> {
+        let st = self.internal.read().unwrap();
+        st.currently_playing.iter().cloned().collect()
+    }
+}
+
+impl Default for App {
+    fn default() -> App {
+        App::new()
+    }
+}

gm-dash/server/src/audio_control.rs

@@ -0,0 +1,138 @@
+use gstreamer::{prelude::*, ClockTime, MessageType, MessageView};
+use std::sync::{Arc, RwLock};
+
+pub struct AudioControl {
+    bus: gstreamer::Bus,
+    pipeline: gstreamer::Pipeline,
+    mixer: gstreamer::Element,
+    audio_sink: gstreamer::Element,
+
+    bus_monitor: std::thread::JoinHandle<()>,
+
+    playing: Arc<RwLock<bool>>,
+}
+
+impl Default for AudioControl {
+    fn default() -> Self {
+        let pipeline = gstreamer::Pipeline::new();
+        let bus = pipeline.bus().unwrap();
+
+        let mixer = gstreamer::ElementFactory::find("audiomixer")
+            .unwrap()
+            .load()
+            .unwrap()
+            .create()
+            .build()
+            .unwrap();
+        pipeline.add(&mixer).unwrap();
+
+        let audio_sink = gstreamer::ElementFactory::find("pulsesink")
+            .unwrap()
+            .load()
+            .unwrap()
+            .create()
+            .build()
+            .unwrap();
+        pipeline.add(&audio_sink).unwrap();
+        mixer.link(&audio_sink).unwrap();
+
+        let playing = Arc::new(RwLock::new(false));
+
+        let bus_monitor = std::thread::spawn({
+            let pipeline_object = pipeline.clone().upcast::<gstreamer::Object>();
+            let playing = playing.clone();
+            let bus = bus.clone();
+            move || loop {
+                if let Some(msg) = bus.timed_pop_filtered(
+                    ClockTime::NONE,
+                    &[
+                        MessageType::Error,
+                        MessageType::Eos,
+                        MessageType::StateChanged,
+                    ],
+                ) {
+                    match msg.view() {
+                        MessageView::StateChanged(st) => {
+                            if msg.src() == Some(&pipeline_object) {
+                                *playing.write().unwrap() = st.current() == gstreamer::State::Playing;
+                            }
+                        }
+                        MessageView::Error(err) => {
+                            println!("error: {:?}", err);
+                        }
+                        MessageView::Eos(_) => {
+                            println!("EOS");
+                        }
+                        _ => {
+                            unreachable!();
+                        }
+                    }
+                }
+            }
+        });
+
+        Self {
+            bus,
+            pipeline,
+            mixer,
+            audio_sink,
+
+            bus_monitor,
+
+            playing,
+        }
+    }
+}
+
+impl AudioControl {
+    pub fn play_pause(&self) {
+        if *self.playing.read().unwrap() {
+            self.pipeline.set_state(gstreamer::State::Paused).unwrap();
+        } else {
+            self.pipeline.set_state(gstreamer::State::Playing).unwrap();
+        }
+    }
+
+    pub fn add_track(&mut self, path: String) {
+        let source = gstreamer::ElementFactory::find("filesrc")
+            .unwrap()
+            .load()
+            .unwrap()
+            .create()
+            .property("location", path)
+            .build()
+            .unwrap();
+        self.pipeline.add(&source).unwrap();
+
+        let decoder = gstreamer::ElementFactory::find("decodebin")
+            .unwrap()
+            .load()
+            .unwrap()
+            .create()
+            .build()
+            .unwrap();
+        self.pipeline.add(&decoder).unwrap();
+        source.link(&decoder).unwrap();
+
+        let volume = gstreamer::ElementFactory::find("volume")
+            .unwrap()
+            .load()
+            .unwrap()
+            .create()
+            .property("mute", false)
+            .property("volume", 0.75)
+            .build()
+            .unwrap();
+        self.pipeline.add(&volume).unwrap();
+        volume.link(&self.mixer).unwrap();
+
+        decoder.connect_pad_added(move |_, pad| {
+            let next_pad = volume.static_pad("sink").unwrap();
+            pad.link(&next_pad).unwrap();
+        });
+    }
+
+    pub fn remove_track(&mut self, path: String) {
+        /* Need to run EOS through to a probe on the trailing end of the volume element */
+    }
+}

gm-dash/server/src/bin/enumerate-audio-sinks.rs

@@ -10,8 +10,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .add_listener_local()
         .global(|global| {
             if global.props.and_then(|p| p.get("media.class")) == Some("Audio/Sink"){
+                // println!("{:?}", global.props.map(|p| p));
                 println!(
-                    "\t{:?} {:?}",
+                    "\t{:?} {:?} {:?}",
+                    global.props.and_then(|p| p.get("node.name")),
                     global.props.and_then(|p| p.get("node.description")),
                     global.props.and_then(|p| p.get("media.class"))
                 );

gm-dash/server/src/bin/play-track.rs

@@ -0,0 +1,200 @@
+use std::time::Duration;
+
+use gstreamer::{
+    prelude::*, Bus, Element, EventType, MessageType, MessageView, Pad, PadDirection, PadPresence,
+    PadProbeData, PadProbeInfo, PadProbeReturn, PadTemplate, Pipeline,
+};
+use pipewire::{context::Context, main_loop::MainLoop};
+
+fn main() {
+    gstreamer::init();
+
+    let pipeline = gstreamer::Pipeline::new();
+    let pipeline_object = pipeline.clone().upcast::<gstreamer::Object>();
+
+    let sinkfactory = gstreamer::ElementFactory::find("pulsesink")
+        .unwrap()
+        .load()
+        .unwrap();
+    let audio_template = sinkfactory
+        .static_pad_templates()
+        .iter()
+        .next()
+        .map(|template| template.get())
+        .unwrap();
+
+    let audio_output = sinkfactory.create().name("sink").build().unwrap();
+    pipeline.add(&audio_output).unwrap();
+
+    let funnel = gstreamer::ElementFactory::find("audiomixer")
+        .unwrap()
+        .load()
+        .unwrap()
+        .create()
+        .build()
+        .unwrap();
+    pipeline.add(&funnel).unwrap();
+
+    let convert = gstreamer::ElementFactory::find("audioconvert")
+        .unwrap()
+        .load()
+        .unwrap()
+        .create()
+        .build()
+        .unwrap();
+    pipeline.add(&convert).unwrap();
+    funnel.link(&convert).unwrap();
+    convert.link(&audio_output).unwrap();
+
+    /*
+    setup_file_reader(
+        &pipeline,
+        funnel.clone(),
+        "/home/savanni/Music/technical-station.ogg",
+    );
+    */
+    setup_file_reader(&pipeline, funnel.clone(), "/home/savanni/Music/techno-city-day.ogg");
+
+    let bus = pipeline.bus().unwrap();
+
+    /*
+    let btsink = sinkfactory
+        .create()
+        .name("sink")
+        .property("device", "bluez_output.0C_A6_94_75_6E_8F.1")
+        .build()
+        .unwrap();
+        */
+
+    pipeline.set_state(gstreamer::State::Playing).unwrap();
+    let pipeline_object = pipeline.clone().upcast::<gstreamer::Object>();
+
+    /*
+    std::thread::spawn({
+        let bus = bus.clone();
+        let pipeline = pipeline.clone();
+        move || {
+            std::thread::sleep(Duration::from_secs(5));
+            swap_audio_output(bus, pipeline, resample, defaultsink, btsink);
+        }
+    });
+    */
+
+    pipeline.set_state(gstreamer::State::Playing).unwrap();
+
+    let mut playing = false;
+
+    loop {
+        if let Some(msg) = bus.timed_pop_filtered(
+            gstreamer::ClockTime::from_mseconds(100),
+            &[
+                MessageType::Error,
+                MessageType::Eos,
+                MessageType::Progress,
+                MessageType::StateChanged,
+                MessageType::StructureChange,
+            ],
+        ) {
+            match msg.view() {
+                MessageView::Progress(prog) => {
+                    println!("progress: {:?}", prog);
+                }
+                MessageView::StateChanged(st) => {
+                    if msg.src() == Some(&pipeline_object) {
+                        println!("State changed from {:?} to {:?}", st.old(), st.current());
+                        playing = st.current() == gstreamer::State::Playing;
+                    }
+                }
+                MessageView::StructureChange(change) => {
+                    println!("structure change: {:?}", change);
+                }
+                _ => {
+                    println!("{:?}", msg);
+                }
+            }
+        } else {
+            if playing {
+                let mut q = gstreamer::query::Position::new(gstreamer::Format::Time);
+                pipeline.query(&mut q);
+                println!("Position result: {:?}", q.result());
+            } else {
+                break;
+            }
+        }
+    }
+
+    pipeline.set_state(gstreamer::State::Null).unwrap();
+}
+
+fn handle_pad_added(element: &Element, pad: &Pad, next_element: &Element, template: &PadTemplate) {
+    println!("handle_pad_added");
+    println!("\t{:?}", element);
+    println!("\t{:?}, {:?}", pad, pad.current_caps());
+    /*
+    let audio_caps = gstreamer::caps::Caps::builder()
+        .field("audio", "audio/x-raw,
+        .build();
+        */
+    /*
+    let audio_pad_template = PadTemplate::new(
+        "audio-pad-template",
+        PadDirection::Sink,
+        PadPresence::Request,
+        &pad.current_caps().unwrap(),
+    )
+    .unwrap();
+    */
+    let next_pad = next_element.request_pad(template, None, None).unwrap();
+    // let converter_pad = converter.static_pad("sink").unwrap();
+    pad.link(&next_pad).unwrap();
+}
+
+fn setup_file_reader(pipeline: &Pipeline, dest: Element, path: &str) {
+    let source = gstreamer::ElementFactory::find("filesrc")
+        .unwrap()
+        .load()
+        .unwrap()
+        .create()
+        .property("location", path)
+        .build()
+        .unwrap();
+
+    let decoder = gstreamer::ElementFactory::find("decodebin")
+        .unwrap()
+        .load()
+        .unwrap()
+        .create()
+        .build()
+        .unwrap();
+
+    let volume = gstreamer::ElementFactory::find("volume")
+        .unwrap()
+        .load()
+        .unwrap()
+        .create()
+        .property("mute", false)
+        .property("volume", 0.5)
+        .build()
+        .unwrap();
+
+    pipeline.add(&source).unwrap();
+    pipeline.add(&decoder).unwrap();
+    pipeline.add(&volume).unwrap();
+
+    source.link(&decoder).unwrap();
+
+    let next_pad = dest.request_pad_simple("sink_%u").unwrap();
+    let volume_output = volume.static_pad("src").unwrap();
+    volume_output.link(&next_pad).unwrap();
+
+    decoder.connect_pad_added(
+        move |element, pad| handle_decoder_started(element, pad, volume.clone())
+    );
+}
+
+fn handle_decoder_started(_: &Element, pad: &Pad, next: Element) {
+    println!("connecting file decoder to converter stream");
+    // let next_pad = next.request_pad_simple("sink_%u").unwrap();
+    let next_pad = next.static_pad("sink").unwrap();
+    pad.link(&next_pad).unwrap();
+}

gm-dash/server/src/main.rs

@@ -1,48 +1,20 @@
 use pipewire::{context::Context, main_loop::MainLoop};
-use std::{
-    net::{Ipv6Addr, SocketAddrV6},
-    sync::{Arc, RwLock},
-};
+use serde::Deserialize;
+use std::net::{Ipv6Addr, SocketAddrV6};
 use tokio::task::spawn_blocking;
 use warp::{serve, Filter};
 
-struct State_ {
-    device_list: Vec<String>,
+mod audio_control;
+
+mod app;
+use app::App;
+
+#[derive(Deserialize)]
+struct PlayTrackParams {
+    track_name: String,
 }
 
-#[derive(Clone)]
-struct State {
-    internal: Arc<RwLock<State_>>,
-}
-
-impl State {
-    fn new() -> State {
-        let internal = State_ {
-            device_list: vec![],
-        };
-        State {
-            internal: Arc::new(RwLock::new(internal)),
-        }
-    }
-
-    fn add_audio(&self, device: String) {
-        let mut st = self.internal.write().unwrap();
-        (*st).device_list.push(device);
-    }
-
-    fn audio_devices(&self) -> Vec<String> {
-        let st = self.internal.read().unwrap();
-        (*st).device_list.clone()
-    }
-}
-
-impl Default for State {
-    fn default() -> State {
-        State::new()
-    }
-}
-
-async fn server_main(state: State) {
+async fn server_main(state: App) {
     let localhost: Ipv6Addr = "::1".parse().unwrap();
     let server_addr = SocketAddrV6::new(localhost, 3001, 0, 0);
 
@@ -55,13 +27,58 @@ async fn server_main(state: State) {
         }
     });
 
-    let routes = root.or(list_output_devices);
+    let list_tracks = warp::path!("tracks").map({
+        let state = state.clone();
+        move || serde_json::to_string(&state.tracks()).unwrap()
+    });
+
+    let play_track = warp::put()
+        .and(warp::path!("playing"))
+        .and(warp::body::json())
+        .map({
+            let state = state.clone();
+            move |params: PlayTrackParams| {
+                state.play(params.track_name);
+                "".to_owned()
+            }
+        });
+
+    let stop_track = warp::delete()
+        .and(warp::path!("playing"))
+        .and(warp::body::json())
+        .map({
+            let state = state.clone();
+            move |params: PlayTrackParams| {
+                state.stop(params.track_name);
+                "".to_owned()
+            }
+        });
+
+    let stop_all_tracks = warp::delete().and(warp::path!("playing")).map({
+        let state = state.clone();
+        move || {
+            state.stop_all();
+            "".to_owned()
+        }
+    });
+
+    let now_playing = warp::path!("playing").map({
+        let state = state.clone();
+        move || serde_json::to_string(&state.playing()).unwrap()
+    });
+
+    let routes = root
+        .or(list_output_devices)
+        .or(list_tracks)
+        .or(play_track)
+        .or(stop_track)
+        .or(stop_all_tracks)
+        .or(now_playing);
 
     serve(routes).run(server_addr).await;
 }
 
-fn handle_add_audio_device(state: State, props: &pipewire::spa::utils::dict::DictRef)
-{
+fn handle_add_audio_device(state: App, props: &pipewire::spa::utils::dict::DictRef) {
     if props.get("media.class") == Some("Audio/Sink") {
         if let Some(device_name) = props.get("node.description") {
             state.add_audio(device_name.to_owned());
@@ -69,7 +86,7 @@ fn handle_add_audio_device(state: State, props: &pipewire::spa::utils::dict::Dic
     }
 }
 
-fn pipewire_loop(state: State) -> Result<(), Box<dyn std::error::Error>> {
+fn pipewire_loop(state: App) -> Result<(), Box<dyn std::error::Error>> {
     let mainloop = MainLoop::new(None)?;
     let context = Context::new(&mainloop)?;
     let core = context.connect(None)?;
@@ -92,13 +109,13 @@ fn pipewire_loop(state: State) -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
-fn pipewire_main(state: State) {
+fn pipewire_main(state: App) {
     pipewire_loop(state).expect("pipewire should not error");
 }
 
 #[tokio::main]
 async fn main() {
-    let state = State::default();
+    let state = App::default();
 
     spawn_blocking({
         let state = state.clone();

rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "1.77.0"
+channel = "1.80.1"
 targets = [ "wasm32-unknown-unknown", "thumbv6m-none-eabi" ]