diff --git a/.gitignore b/.gitignore index f8d4e5d..2c46425 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target .vscode *.flac +*.mp3 bin debug/ diff --git a/Cargo.lock b/Cargo.lock index 997cb4b..8cc2a84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -369,6 +378,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.19" @@ -1342,6 +1370,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mp3lame-encoder" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64622e64e2f9ee2a2fee97a6f53bd8b0301fa0fd15f2f4152f910091949689c7" +dependencies = [ + "libc", + "mp3lame-sys", +] + +[[package]] +name = "mp3lame-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99896fd43ed58e957c164c5f4a1e7d8e770f358835cc6920b2eab7ab6d1a7106" +dependencies = [ + "autotools", + "cc", + "libc", +] + [[package]] name = "mp4ameta" version = "0.11.0" @@ -1697,6 +1746,26 @@ dependencies = [ "rand", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "readme-rustdocifier" version = "0.1.1" @@ -2002,14 +2071,17 @@ dependencies = [ "audiotags", "dirs", "flacenc", + "futures", "indicatif", "lazy_static", "librespot", "machine-uid", + "mp3lame-encoder", "regex", "rpassword 7.3.1", "structopt", "tokio", + "tokio-rayon", "tracing", "tracing-subscriber", ] @@ -2188,6 +2260,16 @@ dependencies = [ "syn 2.0.63", ] +[[package]] +name = "tokio-rayon" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cf33a76e0b1dd03b778f83244137bd59887abf25c0e87bc3e7071105f457693" +dependencies = [ + "rayon", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" diff --git a/Cargo.toml b/Cargo.toml index 8fd3b40..3572809 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,10 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "registry"] } lazy_static = "1.4" async-trait = "0.1" dirs = "5.0" +mp3lame-encoder = { version = "0.1.5", optional = true } +tokio-rayon = "2.1" +futures = "0.3" + +[features] +default = ["mp3"] +mp3 = ["dep:mp3lame-encoder"] \ No newline at end of file diff --git a/src/channel_sink.rs b/src/channel_sink.rs new file mode 100644 index 0000000..ec767df --- /dev/null +++ b/src/channel_sink.rs @@ -0,0 +1,82 @@ +use librespot::playback::audio_backend::Sink; +use librespot::playback::audio_backend::SinkError; +use librespot::playback::convert::Converter; +use librespot::playback::decoder::AudioPacket; + +use crate::track::TrackMetadata; + +pub enum SinkEvent { + Write { bytes: usize, total: usize, content: Vec }, + Finished, +} +pub type SinkEventChannel = tokio::sync::mpsc::UnboundedReceiver; + +pub struct ChannelSink { + sender: tokio::sync::mpsc::UnboundedSender, + bytes_total: usize, + bytes_sent: usize, +} + +impl ChannelSink { + + pub fn new(track: TrackMetadata) -> (Self, SinkEventChannel) { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + ( + ChannelSink { + sender: tx, + bytes_sent: 0, + bytes_total: Self::convert_track_duration_to_size(&track), + }, + rx, + ) + } + + fn convert_track_duration_to_size(metadata: &TrackMetadata) -> usize { + let duration = metadata.duration / 1000; + let sample_rate = 44100; + let channels = 2; + let bits_per_sample = 16; + let bytes_per_sample = bits_per_sample / 8; + (duration as usize) * sample_rate * channels * bytes_per_sample * 2 + } + + pub fn get_approximate_size(&self) -> usize { + self.bytes_total + } +} + +impl Sink for ChannelSink { + fn start(&mut self) -> Result<(), SinkError> { + Ok(()) + } + + fn stop(&mut self) -> Result<(), SinkError> { + tracing::info!("Finished sending song"); + + self.sender + .send(SinkEvent::Finished) + .map_err(|_| SinkError::OnWrite("Failed to send finished event".to_string()))?; + Ok(()) + } + + fn write(&mut self, packet: AudioPacket, converter: &mut Converter) -> Result<(), SinkError> { + let data = converter.f64_to_s16( + packet + .samples() + .map_err(|_| SinkError::OnWrite("Failed to get samples".to_string()))?, + ); + let data32: Vec = data.iter().map(|el| i32::from(*el)).collect(); + self.bytes_sent += data32.len() * std::mem::size_of::(); + + self.sender + .send(SinkEvent::Write { + bytes: self.bytes_sent, + total: self.bytes_total, + content: data32, + }) + .map_err(|_| SinkError::OnWrite("Failed to send event".to_string()))?; + + Ok(()) + } +} diff --git a/src/download.rs b/src/download.rs index e22d6b7..7b3954d 100644 --- a/src/download.rs +++ b/src/download.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::Result; +use futures::StreamExt; +use futures::TryStreamExt; use indicatif::MultiProgress; use indicatif::ProgressBar; use indicatif::ProgressState; @@ -13,8 +15,10 @@ use librespot::playback::mixer::NoOpVolume; use librespot::playback::mixer::VolumeGetter; use librespot::playback::player::Player; -use crate::file_sink::FileSink; -use crate::file_sink::SinkEvent; +use crate::channel_sink::ChannelSink; +use crate::encoder::Format; +use crate::encoder::Samples; +use crate::channel_sink::SinkEvent; use crate::track::Track; use crate::track::TrackMetadata; @@ -29,16 +33,18 @@ pub struct DownloadOptions { pub destination: PathBuf, pub compression: Option, pub parallel: usize, + pub format: Format, } impl DownloadOptions { - pub fn new(destination: Option, compression: Option, parallel: usize) -> Self { + pub fn new(destination: Option, compression: Option, parallel: usize, format: Format) -> Self { let destination = destination.map_or_else(|| std::env::current_dir().unwrap(), PathBuf::from); DownloadOptions { destination, compression, parallel, + format } } } @@ -57,23 +63,13 @@ impl Downloader { tracks: Vec, options: &DownloadOptions, ) -> Result<()> { - let this = Arc::new(self); - - let chunks = tracks.chunks(options.parallel); - for chunk in chunks { - let mut tasks = Vec::new(); - for track in chunk { - let t = track.clone(); - let downloader = this.clone(); - let options = options.clone(); - tasks.push(tokio::spawn(async move { - downloader.download_track(t, &options).await - })); - } - for task in tasks { - task.await??; - } - } + futures::stream::iter(tracks) + .map(|track| { + self.download_track(track, &options) + }) + .buffer_unordered(options.parallel) + .try_collect::>() + .await?; Ok(()) } @@ -87,20 +83,20 @@ impl Downloader { let path = options .destination .join(file_name.clone()) - .with_extension("flac") + .with_extension(options.format.extension()) .to_str() .ok_or(anyhow::anyhow!("Could not set the output path"))? .to_string(); - let (file_sink, mut sink_channel) = FileSink::new(path.to_string(), metadata); + let (sink, mut sink_channel) = ChannelSink::new(metadata); - let file_size = file_sink.get_approximate_size(); + let file_size = sink.get_approximate_size(); let (mut player, _) = Player::new( self.player_config.clone(), self.session.clone(), self.volume_getter(), - move || Box::new(file_sink), + move || Box::new(sink), ); let pb = self.progress_bar.add(ProgressBar::new(file_size as u64)); @@ -111,25 +107,39 @@ impl Downloader { player.load(track.id, true, 0); - let name = file_name.clone(); + let mut samples = Vec::::new(); + tokio::spawn(async move { - while let Some(event) = sink_channel.recv().await { - match event { - SinkEvent::Written { bytes, total } => { - tracing::trace!("Written {} bytes out of {}", bytes, total); - pb.set_position(bytes as u64); - } - SinkEvent::Finished => { - pb.finish_with_message(format!("Downloaded {}", name)); - } - } - } + player.await_end_of_track().await; + player.stop(); }); - player.await_end_of_track().await; - player.stop(); + while let Some(event) = sink_channel.recv().await { + match event { + SinkEvent::Write { bytes, total, mut content } => { + tracing::trace!("Written {} bytes out of {}", bytes, total); + pb.set_position(bytes as u64); + samples.append(&mut content); + } + SinkEvent::Finished => { + pb.finish_with_message(format!("Downloaded {}", &file_name)); + } + } + } tracing::info!("Downloaded track: {:?}", file_name); + + let samples = Samples::new(samples, 44100, 2, 16); + + let format = options.format.clone(); + tokio_rayon::spawn(move || { + tracing::info!("Encoding track: {:?}", file_name); + let encoder = crate::encoder::get_encoder(format).unwrap(); + let stream = encoder.encode(samples).unwrap(); + tracing::info!("Writing track: {:?} to file: {}", file_name, &path); + stream.write_to_file(&path).unwrap(); + }).await; + Ok(()) } diff --git a/src/encoder/flac.rs b/src/encoder/flac.rs new file mode 100644 index 0000000..2337c56 --- /dev/null +++ b/src/encoder/flac.rs @@ -0,0 +1,44 @@ +use flacenc::bitsink::ByteSink; +use flacenc::component::BitRepr; +use flacenc::error::Verified; +use flacenc::error::Verify; + +use super::EncodedStream; +use super::Encoder; +use super::Samples; + +#[derive(Debug)] +pub struct FlacEncoder { + config: Verified, +} + +impl FlacEncoder { + pub fn new() -> anyhow::Result { + let config = flacenc::config::Encoder::default() + .into_verified() + .map_err(|e| anyhow::anyhow!("Failed to verify encoder config: {:?}", e))?; + Ok(FlacEncoder { config }) + } +} + +impl Encoder for FlacEncoder { + fn encode(&self, samples: Samples) -> anyhow::Result { + let source = flacenc::source::MemSource::from_samples( + &samples.samples, + samples.channels as usize, + samples.bits_per_sample as usize, + samples.sample_rate as usize, + ); + + let flac_stream = + flacenc::encode_with_fixed_block_size(&self.config, source, self.config.block_size) + .map_err(|e| anyhow::anyhow!("Failed to encode flac: {:?}", e))?; + + let mut byte_sink = ByteSink::new(); + flac_stream + .write(&mut byte_sink) + .map_err(|e| anyhow::anyhow!("Failed to write flac stream: {:?}", e))?; + + Ok(EncodedStream::new(byte_sink.into_inner())) + } +} diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs new file mode 100644 index 0000000..19fcdc0 --- /dev/null +++ b/src/encoder/mod.rs @@ -0,0 +1,99 @@ +mod flac; +#[cfg(feature = "mp3")] +mod mp3; + +use std::str::FromStr; + +use anyhow::Result; +use lazy_static::lazy_static; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum Format { + Flac, + #[cfg(feature = "mp3")] + Mp3, +} + +impl FromStr for Format { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "flac" => Ok(Format::Flac), + #[cfg(feature = "mp3")] + "mp3" => Ok(Format::Mp3), + _ => Err(anyhow::anyhow!("Unsupported format")), + } + } +} + +impl Format { + + pub fn extension(&self) -> &'static str { + match self { + Format::Flac => "flac", + #[cfg(feature = "mp3")] + Format::Mp3 => "mp3", + } + } + +} + +lazy_static!( + static ref FLAC_ENCODER : Box = Box::new(flac::FlacEncoder::new().unwrap()); + #[cfg(feature = "mp3")] + static ref MP3_ENCODER : Box = Box::new(mp3::Mp3Encoder {}); +); + +pub fn get_encoder(format: Format) -> anyhow::Result<&'static Box> { + match format { + Format::Flac => Ok(&FLAC_ENCODER), + #[cfg(feature = "mp3")] + Format::Mp3 => Ok(&MP3_ENCODER), + _ => Err(anyhow::anyhow!("Unsupported format")), + } +} + +pub trait Encoder { + fn encode(&self, samples: Samples) -> Result; +} + +pub struct Samples { + pub samples: Vec, + pub sample_rate: u32, + pub channels: u32, + pub bits_per_sample: u32, +} + +impl Samples { + pub fn new(samples: Vec, sample_rate: u32, channels: u32, bits_per_sample: u32) -> Self { + Samples { + samples, + sample_rate, + channels, + bits_per_sample, + } + } +} + +pub struct EncodedStream { + pub stream: Vec, +} + +impl EncodedStream { + pub fn new(stream: Vec) -> Self { + EncodedStream { stream } + } + + pub fn write_to_file>(&self, path: P) -> Result<()> { + if !path.as_ref().exists() { + std::fs::create_dir_all( + path.as_ref() + .parent() + .ok_or(anyhow::anyhow!("Could not create path"))?, + )?; + } + std::fs::write(path, &self.stream)?; + Ok(()) + } +} diff --git a/src/encoder/mp3.rs b/src/encoder/mp3.rs new file mode 100644 index 0000000..d920ad9 --- /dev/null +++ b/src/encoder/mp3.rs @@ -0,0 +1,52 @@ +use anyhow::anyhow; +use anyhow::Ok; +use mp3lame_encoder::Builder; +use mp3lame_encoder::FlushNoGap; +use mp3lame_encoder::InterleavedPcm; + +use super::Samples; +use super::Encoder; +use super::EncodedStream; + +pub struct Mp3Encoder; + +impl Mp3Encoder { + fn build_encoder(&self, sample_rate: u32, channels: u32) -> anyhow::Result { + let mut builder = Builder::new(). + ok_or(anyhow::anyhow!("Failed to create mp3 encoder"))?; + + builder.set_sample_rate(sample_rate) + .map_err(|e| anyhow::anyhow!("Failed to set sample rate for mp3 encoder: {}", e))?; + builder.set_num_channels(channels as u8) + .map_err(|e| anyhow::anyhow!("Failed to set number of channels for mp3 encoder: {}", e))?; + builder.set_brate(mp3lame_encoder::Birtate::Kbps160) + .map_err(|e| anyhow::anyhow!("Failed to set bitrate for mp3 encoder: {}", e))?; + + builder.build() + .map_err(|e| anyhow::anyhow!("Failed to build mp3 encoder: {}", e)) + } +} + +impl Encoder for Mp3Encoder { + fn encode(&self, samples: Samples) -> anyhow::Result { + let mut mp3_encoder = self.build_encoder(samples.sample_rate, samples.channels)?; + + let samples: Vec = samples.samples.iter().map(|&x| x as i16).collect(); + let input = InterleavedPcm(samples.as_slice()); + let mut mp3_out_buffer = Vec::new(); + mp3_out_buffer.reserve(mp3lame_encoder::max_required_buffer_size(samples.len())); + let encoded_size = mp3_encoder.encode(input, mp3_out_buffer.spare_capacity_mut()) + .map_err(|e| anyhow!("Failed to encode mp3: {}", e))?; + unsafe { + mp3_out_buffer.set_len(mp3_out_buffer.len().wrapping_add(encoded_size)); + } + + let encoded_size = mp3_encoder.flush::(mp3_out_buffer.spare_capacity_mut()) + .map_err(|e| anyhow!("Failed to flush mp3 encoder: {}", e))?; + unsafe { + mp3_out_buffer.set_len(mp3_out_buffer.len().wrapping_add(encoded_size)); + } + + Ok(EncodedStream::new(mp3_out_buffer)) + } +} \ No newline at end of file diff --git a/src/file_sink.rs b/src/file_sink.rs index 50e6407..395ec2c 100644 --- a/src/file_sink.rs +++ b/src/file_sink.rs @@ -9,6 +9,8 @@ use librespot::playback::audio_backend::SinkError; use librespot::playback::convert::Converter; use librespot::playback::decoder::AudioPacket; +use crate::encoder::get_encoder; +use crate::encoder::Samples; use crate::track::TrackMetadata; pub enum SinkEvent { @@ -67,18 +69,42 @@ impl Sink for FileSink { fn stop(&mut self) -> Result<(), SinkError> { tracing::info!("Writing to file: {:?}", &self.sink); - let config = flacenc::config::Encoder::default() - .into_verified() - .map_err(|_| SinkError::OnWrite("Failed to create flac encoder".to_string()))?; - let source = flacenc::source::MemSource::from_samples(&self.content, 2, 16, 44100); - let flac_stream = flacenc::encode_with_fixed_block_size(&config, source, config.block_size) - .map_err(|_| SinkError::OnWrite("Failed to encode flac".to_string()))?; - let mut sink = flacenc::bitsink::ByteSink::new(); - flac_stream - .write(&mut sink) - .map_err(|_| SinkError::OnWrite("Failed to write flac to sink".to_string()))?; - std::fs::write(&self.sink, sink.as_slice()) + // let config = flacenc::config::Encoder::default() + // .into_verified() + // .map_err(|_| SinkError::OnWrite("Failed to create flac encoder".to_string()))?; + // let source = flacenc::source::MemSource::from_samples(&self.content, 2, 16, 44100); + // let flac_stream = flacenc::encode_with_fixed_block_size(&config, source, config.block_size) + // .map_err(|_| SinkError::OnWrite("Failed to encode flac".to_string()))?; + // let mut sink = flacenc::bitsink::ByteSink::new(); + // flac_stream + // .write(&mut sink) + // .map_err(|_| SinkError::OnWrite("Failed to write flac to sink".to_string()))?; + // std::fs::write(&self.sink, sink.as_slice()) + // .map_err(|_| SinkError::OnWrite("Failed to write flac to file".to_string()))?; + let flac_enc = get_encoder(crate::encoder::Format::Flac) + .map_err(|_| SinkError::OnWrite("Failed to get flac encoder".to_string()))?; + let mp3_enc = get_encoder(crate::encoder::Format::Mp3) + .map_err(|_| SinkError::OnWrite("Failed to get mp3 encoder".to_string()))?; + + let flac_stream = flac_enc.encode(Samples::new( + self.content.clone(), + 44100, + 2, + 16, + )) + .map_err(|_| SinkError::OnWrite("Failed to encode flac".to_string()))?; + let mp3_stream = mp3_enc.encode(Samples::new( + self.content.clone(), + 44100, + 2, + 16, + )) + .map_err(|_| SinkError::OnWrite("Failed to encode mp3".to_string()))?; + + flac_stream.write_to_file(&self.sink) .map_err(|_| SinkError::OnWrite("Failed to write flac to file".to_string()))?; + mp3_stream.write_to_file(&self.sink) + .map_err(|_| SinkError::OnWrite("Failed to write mp3 to file".to_string()))?; let mut tag = Tag::new() .with_tag_type(TagType::Flac) diff --git a/src/lib.rs b/src/lib.rs index 8738005..f150af9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,6 @@ pub mod download; pub mod file_sink; +pub mod channel_sink; pub mod session; pub mod track; +pub mod encoder; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 5b96e1b..389b35a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use spotify_dl::download::{DownloadOptions, Downloader}; +use spotify_dl::encoder::Format; use spotify_dl::session::create_session; use spotify_dl::track::get_tracks; use structopt::StructOpt; @@ -41,6 +42,13 @@ struct Opt { default_value = "5" )] parallel: usize, + #[structopt( + short = "f", + long = "format", + help = "The format to download the tracks in. Default is flac.", + default_value = "flac" + )] + format: Format } pub fn configure_logger() { @@ -84,7 +92,7 @@ async fn main() -> anyhow::Result<()> { downloader .download_tracks( track, - &DownloadOptions::new(opt.destination, opt.compression, opt.parallel), + &DownloadOptions::new(opt.destination, opt.compression, opt.parallel, opt.format), ) .await } diff --git a/src/track.rs b/src/track.rs index 5f3ff85..c96dac9 100644 --- a/src/track.rs +++ b/src/track.rs @@ -14,6 +14,7 @@ trait TrackCollection { pub async fn get_tracks(spotify_ids: Vec, session: &Session) -> Result> { let mut tracks: Vec = Vec::new(); for id in spotify_ids { + tracing::debug!("Getting tracks for: {}", id); let id = parse_uri_or_url(&id).ok_or(anyhow::anyhow!("Invalid track"))?; let new_tracks = match id.audio_type { librespot::core::spotify_id::SpotifyAudioType::Track => vec![Track::from_id(id)], @@ -39,7 +40,9 @@ fn parse_uri_or_url(track: &str) -> Option { } fn parse_uri(track_uri: &str) -> Option { - SpotifyId::from_uri(track_uri).ok() + let res = SpotifyId::from_uri(track_uri); + tracing::info!("Parsed URI: {:?}", res); + res.ok() } fn parse_url(track_url: &str) -> Option {