From aed74fbb8a58d89000fbc6fe5e1315b70d13ed4b Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Tue, 5 Dec 2023 21:47:44 +0100 Subject: [PATCH 01/12] [xcode] Support for ALAC encoded RSP/DAAP streaming Also changes WAV encoding to use source quality instead of fixed 44100/16/2. Sets the default to WAV, since it has the best quality, and doesn't have the delay that creating an MP4 container for ALAC currently has (TODO prepare the headers to avoid the delay). Ref issue #1182 and #1665 --- owntone.conf.in | 21 +- src/httpd.c | 29 +- src/httpd_daap.c | 7 +- src/httpd_internal.h | 5 - src/httpd_rsp.c | 17 +- src/transcode.c | 654 +++++++++++++++++++++++++++++++------------ src/transcode.h | 23 +- 7 files changed, 548 insertions(+), 208 deletions(-) diff --git a/owntone.conf.in b/owntone.conf.in index 1c1ad7b3..30d67b19 100644 --- a/owntone.conf.in +++ b/owntone.conf.in @@ -187,19 +187,24 @@ library { # Should we import the content of iTunes smart playlists? # itunes_smartpl = false - # Decoding options for DAAP and RSP clients + # Transcoding options for DAAP and RSP clients # Since iTunes has native support for mpeg, mp4a, mp4v, alac and wav, - # such files will be sent as they are. Any other formats will be decoded - # to raw wav. If OwnTone detects a non-iTunes DAAP client, it is - # assumed to only support mpeg and wav, other formats will be decoded. - # Here you can change when to decode. Note that these settings only - # affect serving media to DAAP and RSP clients, they have no effect on + # such files will be sent as they are. Any other formats will be + # transcoded. Some other clients, including Roku/RSP, announce what + # formats they support, and the server will transcode to one of those if + # necessary. Clients that don't announce supported formats are assumed + # to support mpeg (mp3), wav and alac. + # Here you can change when and how to transcode. The settings *only* + # affect serving audio to DAAP and RSP clients, they have no effect on # direct AirPlay, Chromecast and local audio playback. # Formats: mp4a, mp4v, mpeg, alac, flac, mpc, ogg, wma, wmal, wmav, aif, wav - # Formats that should never be decoded + # Formats that should never be transcoded # no_decode = { "format", "format" } - # Formats that should always be decoded + # Formats that should always be transcoded # force_decode = { "format", "format" } + # Prefer transcode to wav (default), alac or mpeg (mp3 with the bit rate + # configured below in the streaming section) +# prefer_format = "format" # Set ffmpeg filters (similar to 'ffmpeg -af xxx') that you want the # server to use when decoding files from your library. Examples: diff --git a/src/httpd.c b/src/httpd.c index be7f2301..65a50720 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -105,18 +105,19 @@ struct stream_ctx { static const struct content_type_map ext2ctype[] = { - { ".html", XCODE_NONE, "text/html; charset=utf-8" }, - { ".xml", XCODE_NONE, "text/xml; charset=utf-8" }, - { ".css", XCODE_NONE, "text/css; charset=utf-8" }, - { ".txt", XCODE_NONE, "text/plain; charset=utf-8" }, - { ".js", XCODE_NONE, "application/javascript; charset=utf-8" }, - { ".gif", XCODE_NONE, "image/gif" }, - { ".ico", XCODE_NONE, "image/x-ico" }, - { ".png", XCODE_PNG, "image/png" }, - { ".jpg", XCODE_JPEG, "image/jpeg" }, - { ".mp3", XCODE_MP3, "audio/mpeg" }, - { ".wav", XCODE_WAV, "audio/wav" }, - { NULL, XCODE_NONE, NULL } + { ".html", XCODE_NONE, "text/html; charset=utf-8" }, + { ".xml", XCODE_NONE, "text/xml; charset=utf-8" }, + { ".css", XCODE_NONE, "text/css; charset=utf-8" }, + { ".txt", XCODE_NONE, "text/plain; charset=utf-8" }, + { ".js", XCODE_NONE, "application/javascript; charset=utf-8" }, + { ".gif", XCODE_NONE, "image/gif" }, + { ".ico", XCODE_NONE, "image/x-ico" }, + { ".png", XCODE_PNG, "image/png" }, + { ".jpg", XCODE_JPEG, "image/jpeg" }, + { ".mp3", XCODE_MP3, "audio/mpeg" }, + { ".m4a", XCODE_MP4_ALAC, "audio/mp4" }, + { ".wav", XCODE_WAV, "audio/wav" }, + { NULL, XCODE_NONE, NULL } }; static char webroot_directory[PATH_MAX]; @@ -672,8 +673,8 @@ static struct stream_ctx * stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile, struct httpd_request *hreq, int64_t offset, int64_t end_offset, event_callback_fn stream_cb) { + struct media_quality quality = { 0 }; struct stream_ctx *st; - struct media_quality quality = { HTTPD_STREAM_SAMPLE_RATE, HTTPD_STREAM_BPS, HTTPD_STREAM_CHANNELS, HTTPD_STREAM_BIT_RATE }; st = stream_new(mfi, hreq, stream_cb); if (!st) @@ -681,6 +682,8 @@ stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile goto error; } + // We use source sample rate etc, but for MP3 we must set a bit rate + quality.bit_rate = cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); st->xcode = transcode_setup(profile, &quality, mfi->data_kind, mfi->path, mfi->song_length); if (!st->xcode) { diff --git a/src/httpd_daap.c b/src/httpd_daap.c index e5238938..20f753cf 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -1151,7 +1151,7 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) size_t len; enum transcode_profile profile; struct transcode_metadata_string xcode_metadata; - struct media_quality quality = { HTTPD_STREAM_SAMPLE_RATE, HTTPD_STREAM_BPS, HTTPD_STREAM_CHANNELS, HTTPD_STREAM_BIT_RATE }; + struct media_quality quality = { 0 }; uint32_t len_ms; int nmeta = 0; int sort_headers; @@ -1239,6 +1239,11 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) if (safe_atou32(dbmfi.song_length, &len_ms) < 0) len_ms = 3 * 60 * 1000; // just a fallback default + safe_atoi32(dbmfi.samplerate, &quality.sample_rate); + safe_atoi32(dbmfi.bits_per_sample, &quality.bits_per_sample); + safe_atoi32(dbmfi.channels, &quality.channels); + quality.bit_rate = cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); + transcode_metadata_strings_set(&xcode_metadata, profile, &quality, len_ms); dbmfi.type = xcode_metadata.type; dbmfi.codectype = xcode_metadata.codectype; diff --git a/src/httpd_internal.h b/src/httpd_internal.h index 62e285b2..28262572 100644 --- a/src/httpd_internal.h +++ b/src/httpd_internal.h @@ -36,11 +36,6 @@ #define HTTP_BADGATEWAY 502 /**< received an invalid response from the upstream */ #define HTTP_SERVUNAVAIL 503 /**< the server is not available */ -#define HTTPD_STREAM_SAMPLE_RATE 44100 -#define HTTPD_STREAM_BPS 16 -#define HTTPD_STREAM_CHANNELS 2 -#define HTTPD_STREAM_BIT_RATE 320000 - struct httpd_request; diff --git a/src/httpd_rsp.c b/src/httpd_rsp.c index 2ea40213..4e53b067 100644 --- a/src/httpd_rsp.c +++ b/src/httpd_rsp.c @@ -417,7 +417,7 @@ rsp_reply_db(struct httpd_request *hreq) static int item_add(xml_node *parent, struct query_params *qp, const char *user_agent, const char *client_codecs, int mode) { - struct media_quality quality = { HTTPD_STREAM_SAMPLE_RATE, HTTPD_STREAM_BPS, HTTPD_STREAM_CHANNELS, HTTPD_STREAM_BIT_RATE }; + struct media_quality quality = { 0 }; struct db_media_file_info dbmfi; struct transcode_metadata_string xcode_metadata; enum transcode_profile profile; @@ -444,6 +444,11 @@ item_add(xml_node *parent, struct query_params *qp, const char *user_agent, cons if (safe_atou32(dbmfi.song_length, &len_ms) < 0) len_ms = 3 * 60 * 1000; // just a fallback default + safe_atoi32(dbmfi.samplerate, &quality.sample_rate); + safe_atoi32(dbmfi.bits_per_sample, &quality.bits_per_sample); + safe_atoi32(dbmfi.channels, &quality.channels); + quality.bit_rate = cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); + transcode_metadata_strings_set(&xcode_metadata, profile, &quality, len_ms); dbmfi.type = xcode_metadata.type; dbmfi.codectype = xcode_metadata.codectype; @@ -715,6 +720,16 @@ rsp_stream(struct httpd_request *hreq) // /rsp/stream/36364 // /rsp/db/0?query=id%3D36365&type=full // /rsp/stream/36365 +// +// Headers sent from Roku M2000 and M1001 in stream requests (and other?): +// +// 'User-Agent': 'Roku SoundBridge/3.0' +// 'Host': '192.168.1.119:3689' +// 'Accept': '*/*' +// 'Pragma': 'no-cache' +// 'accept-codecs': 'wma,mpeg,wav,mp4a,alac' +// 'rsp-version': '0.1' +// 'transcode-codecs': 'wav,mp3' static struct httpd_uri_map rsp_handlers[] = { { diff --git a/src/transcode.c b/src/transcode.c index 6dc9308d..632df7df 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -20,6 +20,8 @@ # include #endif +#define _GNU_SOURCE // For memmem() + #include #include #include @@ -64,14 +66,19 @@ #define WAV_HEADER_LEN 44 // Max filters in a filtergraph #define MAX_FILTERS 9 +// Set to same size as in httpd.c (but can be set to something else) +#define STREAM_CHUNK_SIZE (64 * 1024) -static const char *default_codecs = "mpeg,wav"; +static const char *default_codecs = "mpeg,alac,wav"; static const char *roku_codecs = "mpeg,mp4a,wma,alac,wav"; static const char *itunes_codecs = "mpeg,mp4a,mp4v,alac,wav"; // Used for passing errors to DPRINTF (can't count on av_err2str being present) static char errbuf[64]; +// Used by dummy_seek to mark a seek requested by ffmpeg +static const uint8_t xcode_seek_marker[8] = { 0x0D, 0x0E, 0x0A, 0x0D, 0x0B, 0x0E, 0x0E, 0x0F }; + // The settings struct will be filled out based on the profile enum struct settings_ctx { @@ -94,12 +101,15 @@ struct settings_ctx AVChannelLayout channel_layout; #else uint64_t channel_layout; - int channels; #endif + int nb_channels; int bit_rate; int frame_size; enum AVSampleFormat sample_format; + bool with_mp4_header; bool with_wav_header; + bool without_libav_header; + bool without_libav_trailer; bool with_icy; bool with_user_filters; @@ -180,6 +190,9 @@ struct encode_ctx // The ffmpeg muxer writes to this buffer using the avio_evbuffer interface struct evbuffer *obuf; + // IO Context for non-file output + struct transcode_evbuf_io evbuf_io; + // Contains the most recent packet from av_buffersink_get_frame() AVFrame *filt_frame; @@ -195,9 +208,6 @@ struct encode_ctx // Used to check for ICY metadata changes at certain intervals uint32_t icy_interval; uint32_t icy_hash; - - // WAV header - uint8_t wav_header[WAV_HEADER_LEN]; }; enum probe_type @@ -290,6 +300,21 @@ init_settings(struct settings_ctx *settings, enum transcode_profile profile, str settings->frame_size = 352; break; + case XCODE_MP4_ALAC: + settings->with_mp4_header = true; + settings->encode_audio = true; + settings->format = "data"; + settings->audio_codec = AV_CODEC_ID_ALAC; + break; + + case XCODE_MP4_ALAC_HEADER: + settings->without_libav_header = true; + settings->without_libav_trailer = true; + settings->encode_audio = true; + settings->format = "ipod"; // ffmpeg default mp4 variant ("mp4" doesn't work with SoundBridge because of the btrt atom in the header) + settings->audio_codec = AV_CODEC_ID_ALAC; + break; + case XCODE_OGG: settings->encode_audio = true; settings->in_format = "ogg"; @@ -354,7 +379,7 @@ init_settings(struct settings_ctx *settings, enum transcode_profile profile, str av_channel_layout_default(&settings->channel_layout, quality->channels); #else settings->channel_layout = av_get_default_channel_layout(quality->channels); - settings->channels = quality->channels; + settings->nb_channels = quality->channels; #endif } @@ -372,6 +397,66 @@ init_settings(struct settings_ctx *settings, enum transcode_profile profile, str return 0; } +static int +init_settings_from_video(struct settings_ctx *settings, enum transcode_profile profile, struct decode_ctx *src_ctx, int width, int height) +{ + settings->width = width; + settings->height = height; + + return 0; +} + +static int +init_settings_from_audio(struct settings_ctx *settings, enum transcode_profile profile, struct decode_ctx *src_ctx, struct media_quality *quality) +{ + int src_bytes_per_sample = av_get_bytes_per_sample(src_ctx->audio_stream.codec->sample_fmt); + + // Initialize unset settings that are source-dependent, not profile-dependent + if (!settings->sample_rate) + settings->sample_rate = src_ctx->audio_stream.codec->sample_rate; + +#if USE_CH_LAYOUT + if (!av_channel_layout_check(&settings->channel_layout)) + av_channel_layout_copy(&settings->channel_layout, &src_ctx->audio_stream.codec->ch_layout); + + settings->nb_channels = settings->channel_layout.nb_channels; +#else + if (settings->nb_channels == 0) + { + settings->nb_channels = src_ctx->audio_stream.codec->channels; + settings->channel_layout = src_ctx->audio_stream.codec->channel_layout; + } +#endif + + // Initialize settings that are both source-dependent and profile-dependent + switch (profile) + { + case XCODE_MP4_ALAC: + case XCODE_MP4_ALAC_HEADER: + if (!settings->sample_format) + settings->sample_format = (src_bytes_per_sample == 4) ? AV_SAMPLE_FMT_S32P : AV_SAMPLE_FMT_S16P; + break; + + case XCODE_PCM_NATIVE: + if (!settings->sample_format) + settings->sample_format = (src_bytes_per_sample == 4) ? AV_SAMPLE_FMT_S32 : AV_SAMPLE_FMT_S16; + if (!settings->audio_codec) + settings->audio_codec = (src_bytes_per_sample == 4) ? AV_CODEC_ID_PCM_S32LE : AV_CODEC_ID_PCM_S16LE; + if (!settings->format) + settings->format = (src_bytes_per_sample == 4) ? "s32le" : "s16le"; + break; + + default: + if (settings->sample_format && settings->audio_codec && settings->format) + return 0; + + DPRINTF(E_LOG, L_XCODE, "Bug! Profile %d has unset encoding parameters\n", profile); + return -1; + } + + return 0; +} + static void stream_settings_set(struct stream_ctx *s, struct settings_ctx *settings, enum AVMediaType type) { @@ -382,7 +467,7 @@ stream_settings_set(struct stream_ctx *s, struct settings_ctx *settings, enum AV av_channel_layout_copy(&s->codec->ch_layout, &(settings->channel_layout)); #else s->codec->channel_layout = settings->channel_layout; - s->codec->channels = settings->channels; + s->codec->channels = settings->nb_channels; #endif s->codec->sample_fmt = settings->sample_format; s->codec->time_base = (AVRational){1, settings->sample_rate}; @@ -436,26 +521,47 @@ add_le32(uint8_t *dst, uint32_t val) dst[3] = (val >> 24) & 0xff; } -/* - * header must have size WAV_HEADER_LEN (44 bytes) - */ -static void -make_wav_header(uint8_t *header, int sample_rate, int bytes_per_sample, int channels, off_t bytes_total) +// Copies the src buffer to position pos of the dst buffer, expanding dst if +// needed to fit src. Can be called with *dst = NULL and *dst_len = 0. Returns +// the number of bytes dst was expanded with. +static int +copy_buffer_to_position(uint8_t **dst, size_t *dst_len, uint8_t *src, size_t src_len, int64_t pos) { - uint32_t wav_size = bytes_total - WAV_HEADER_LEN; + int bytes_added = 0; - memcpy(header, "RIFF", 4); - add_le32(header + 4, 36 + wav_size); - memcpy(header + 8, "WAVEfmt ", 8); - add_le32(header + 16, 16); - add_le16(header + 20, 1); - add_le16(header + 22, channels); /* channels */ - add_le32(header + 24, sample_rate); /* samplerate */ - add_le32(header + 28, sample_rate * channels * bytes_per_sample); /* byte rate */ - add_le16(header + 32, channels * bytes_per_sample); /* block align */ - add_le16(header + 34, 8 * bytes_per_sample); /* bits per sample */ - memcpy(header + 36, "data", 4); - add_le32(header + 40, wav_size); + if (pos < 0 || pos > *dst_len) + return -1; // Out of bounds + if (src_len == 0) + return 0; // Nothing to do + + if (pos + src_len > *dst_len) + { + bytes_added = pos + src_len - *dst_len; + *dst_len += bytes_added; + CHECK_NULL(L_XCODE, *dst = realloc(*dst, *dst_len)); + } + + memcpy(*dst + pos, src, src_len); + return bytes_added; +} + +// Doesn't actually seek, just inserts a marker in the obuf +static int64_t +dummy_seek(void *arg, int64_t offset, enum transcode_seek_type type) +{ + struct transcode_ctx *ctx = arg; + struct encode_ctx *enc_ctx = ctx->encode_ctx; + + if (type == XCODE_SEEK_SET) + { + evbuffer_add(enc_ctx->obuf, xcode_seek_marker, sizeof(xcode_seek_marker)); + evbuffer_add(enc_ctx->obuf, &offset, sizeof(offset)); + return offset; + } + else if (type == XCODE_SEEK_SIZE) + return enc_ctx->bytes_total; + + return -1; } static off_t @@ -475,6 +581,8 @@ size_estimate(enum transcode_profile profile, int bit_rate, int sample_rate, int bytes = (int64_t)len_ms * channels * bytes_per_sample * sample_rate / 1000 + WAV_HEADER_LEN; else if (profile == XCODE_MP3) bytes = (int64_t)len_ms * bit_rate / 8000; + else if (profile == XCODE_MP4_ALAC) + bytes = (int64_t)len_ms * channels * bytes_per_sample * sample_rate / 1000 / 2; // FIXME else bytes = -1; @@ -841,6 +949,7 @@ static int read_decode_filter_encode_write(struct transcode_ctx *ctx) { struct decode_ctx *dec_ctx = ctx->decode_ctx; + struct encode_ctx *enc_ctx = ctx->encode_ctx; enum AVMediaType type; int ret; @@ -855,12 +964,10 @@ read_decode_filter_encode_write(struct transcode_ctx *ctx) if (dec_ctx->video_stream.stream) decode_filter_encode_write(ctx, &dec_ctx->video_stream, NULL, AVMEDIA_TYPE_VIDEO); - // Flush muxer - if (ctx->encode_ctx) - { - av_interleaved_write_frame(ctx->encode_ctx->ofmt_ctx, NULL); - av_write_trailer(ctx->encode_ctx->ofmt_ctx); - } + if (enc_ctx) + av_interleaved_write_frame(enc_ctx->ofmt_ctx, NULL); // Flush muxer + if (enc_ctx && !enc_ctx->settings.without_libav_trailer) + av_write_trailer(enc_ctx->ofmt_ctx); return ret; } @@ -951,7 +1058,7 @@ avio_evbuffer_open(struct transcode_evbuf_io *evbuf_io, int is_output) ae->seekfn_arg = evbuf_io->seekfn_arg; if (is_output) - s = avio_alloc_context(ae->buffer, AVIO_BUFFER_SIZE, 1, ae, NULL, avio_evbuffer_write, NULL); + s = avio_alloc_context(ae->buffer, AVIO_BUFFER_SIZE, 1, ae, NULL, avio_evbuffer_write, (evbuf_io->seekfn ? avio_evbuffer_seek : NULL)); else s = avio_alloc_context(ae->buffer, AVIO_BUFFER_SIZE, 0, ae, avio_evbuffer_read, NULL, (evbuf_io->seekfn ? avio_evbuffer_seek : NULL)); @@ -969,22 +1076,6 @@ avio_evbuffer_open(struct transcode_evbuf_io *evbuf_io, int is_output) return s; } -static AVIOContext * -avio_input_evbuffer_open(struct transcode_evbuf_io *evbuf_io) -{ - return avio_evbuffer_open(evbuf_io, 0); -} - -static AVIOContext * -avio_output_evbuffer_open(struct evbuffer *evbuf) -{ - struct transcode_evbuf_io evbuf_io = { 0 }; - - evbuf_io.evbuf = evbuf; - - return avio_evbuffer_open(&evbuf_io, 1); -} - static void avio_evbuffer_close(AVIOContext *s) { @@ -1004,6 +1095,220 @@ avio_evbuffer_close(AVIOContext *s) } +/* ----------------------- CUSTOM HEADER GENERATION ------------------------ */ + +static int +make_wav_header(struct evbuffer **wav_header, int sample_rate, int bytes_per_sample, int channels, off_t bytes_total) +{ + uint8_t header[WAV_HEADER_LEN]; + + uint32_t wav_size = bytes_total - WAV_HEADER_LEN; + + memcpy(header, "RIFF", 4); + add_le32(header + 4, 36 + wav_size); + memcpy(header + 8, "WAVEfmt ", 8); + add_le32(header + 16, 16); + add_le16(header + 20, 1); + add_le16(header + 22, channels); /* channels */ + add_le32(header + 24, sample_rate); /* samplerate */ + add_le32(header + 28, sample_rate * channels * bytes_per_sample); /* byte rate */ + add_le16(header + 32, channels * bytes_per_sample); /* block align */ + add_le16(header + 34, 8 * bytes_per_sample); /* bits per sample */ + memcpy(header + 36, "data", 4); + add_le32(header + 40, wav_size); + + *wav_header = evbuffer_new(); + evbuffer_add(*wav_header, header, sizeof(header)); + return 0; +} + +static int +mp4_adjust_moov_stco_offset(uint8_t *moov, size_t moov_len) +{ + uint8_t stco_needle[8] = { 's', 't', 'c', 'o', 0, 0, 0, 0 }; + uint32_t be32; + uint32_t n_entries; + uint32_t entry; + uint8_t *ptr; + uint8_t *end; + + end = moov + moov_len; + ptr = memmem(moov, moov_len, stco_needle, sizeof(stco_needle)); + if (!ptr || ptr + sizeof(stco_needle) + sizeof(be32) > end) + return -1; + + ptr += sizeof(stco_needle); + memcpy(&be32, ptr, sizeof(be32)); + for (n_entries = be32toh(be32); n_entries > 0; n_entries--) + { + ptr += sizeof(be32); + if (ptr + sizeof(be32) > end) + return -1; + + memcpy(&be32, ptr, sizeof(be32)); + entry = be32toh(be32); + be32 = htobe32(entry + moov_len); + memcpy(ptr, &be32, sizeof(be32)); + } + + return 0; +} + +static int +mp4_header_trailer_from_evbuf(uint8_t **header, size_t *header_len, uint8_t **trailer, size_t *trailer_len, struct evbuffer *evbuf, int64_t start_pos) +{ + uint8_t *buf = evbuffer_pullup(evbuf, -1); + size_t buf_len = evbuffer_get_length(evbuf); + int64_t pos = start_pos; + int bytes_added = 0; + uint8_t *marker; + size_t len; + int ret; + + while (buf_len > 0) + { + marker = memmem(buf, buf_len, xcode_seek_marker, sizeof(xcode_seek_marker)); + len = marker ? marker - buf : buf_len; + + if (pos <= *header_len) // Either first write of header or seek to pos inside header + ret = copy_buffer_to_position(header, header_len, buf, len, pos); + else if (pos >= start_pos) // Either first write of trailer or seek to pos inside trailer + ret = copy_buffer_to_position(trailer, trailer_len, buf, len, pos - start_pos); + else // Unexpected seek to body (pos is before trailer but not in header) + ret = -1; + + if (ret < 0) + return -1; + + bytes_added += ret; + if (!marker) + break; + + memcpy(&pos, marker + sizeof(xcode_seek_marker), sizeof(pos)); + buf += len + sizeof(xcode_seek_marker) + sizeof(pos); + buf_len -= len + sizeof(xcode_seek_marker) + sizeof(pos); + } + + evbuffer_drain(evbuf, -1); + return bytes_added; +} + +// Transcodes the entire file so that we can grab the header, which will then +// have a correct moov atom. The moov atom contains elements like stco and stsz +// which can only be made when the encoding has been done, since they contain +// information about where the frames are in the file. iTunes and Soundsbrdige +// requires these to be correct, otherwise they won't play our transcoded files. +// They also require that the atom is in the beginning of the file. ffmpeg's +// "faststart" option does this, but is difficult to use with non-file output, +// instead we move the atom ourselves. +static int +make_mp4_header(struct evbuffer **mp4_header, const char *url) +{ + struct transcode_ctx ctx = { 0 }; + struct transcode_evbuf_io evbuf_io = { 0 }; + uint8_t free_tag[4] = { 'f', 'r', 'e', 'e' }; + uint8_t *header = NULL; + uint8_t *trailer = NULL; + size_t header_len = 0; + size_t trailer_len = 0; + uint8_t *ptr; + int ret; + + if (!url || *url != '/') + return -1; + + CHECK_NULL(L_XCODE, evbuf_io.evbuf = evbuffer_new()); + + evbuf_io.seekfn = dummy_seek; + evbuf_io.seekfn_arg = &ctx; + + ctx.decode_ctx = transcode_decode_setup(XCODE_MP4_ALAC_HEADER, NULL, DATA_KIND_FILE, url, NULL, -1); + if (!ctx.decode_ctx) + goto error; + + ctx.encode_ctx = transcode_encode_setup_with_io(XCODE_MP4_ALAC_HEADER, NULL, &evbuf_io, ctx.decode_ctx, 0, 0); + if (!ctx.encode_ctx) + goto error; + + // Save the template header, which looks something like this (note that the + // mdate size is still unknown, so just zeroes, and there is no moov): + // + // 0000 00 00 00 1c 66 74 79 70 69 73 6f 6d 00 00 02 00 ....ftypisom.... + // 0010 69 73 6f 6d 69 73 6f 32 6d 70 34 31 00 00 00 08 isomiso2mp41.... + // 0020 66 72 65 65 00 00 00 00 6d 64 61 74 free....mdat + ret = avformat_write_header(ctx.encode_ctx->ofmt_ctx, NULL); + if (ret < 0) + goto error; + + // Writes the obuf to the header buffer, bytes_processed is 0 + ret = mp4_header_trailer_from_evbuf(&header, &header_len, &trailer, &trailer_len, ctx.encode_ctx->obuf, ctx.encode_ctx->bytes_processed); + if (ret < 0) + goto error; + + ctx.encode_ctx->bytes_processed += ret; + + // Encode but discard result, this is just so that ffmpeg can create the + // missing header data. + while (read_decode_filter_encode_write(&ctx) == 0) + { + ctx.encode_ctx->bytes_processed += evbuffer_get_length(ctx.encode_ctx->obuf); + evbuffer_drain(ctx.encode_ctx->obuf, -1); + } + + // Here, ffmpeg will seek back and write the size to the mdat atom and then + // seek forward again to write the trailer. Since we can't actually seek, we + // instead look for the markers that dummy_seek() inserted. + av_write_trailer(ctx.encode_ctx->ofmt_ctx); + ret = mp4_header_trailer_from_evbuf(&header, &header_len, &trailer, &trailer_len, ctx.encode_ctx->obuf, ctx.encode_ctx->bytes_processed); + if (ret < 0 || !header || !trailer) + goto error; + + // The trailer buffer should now contain the moov atom. We need to adjust the + // chunk offset (stco) in it because we will move it to the beginning of the + // file. + ret = mp4_adjust_moov_stco_offset(trailer, trailer_len); + if (ret < 0) + goto error; + + // Now we want to move the trailer (which has the moov atom) into the header. + // We insert it before the free atom, because that's what ffmpeg does when + // the "faststart" option is set. + CHECK_NULL(L_XCODE, header = realloc(header, header_len + trailer_len)); + + ptr = memmem(header, header_len, free_tag, sizeof(free_tag)); + if (!ptr || ptr - header < sizeof(uint32_t)) + goto error; + + ptr -= sizeof(uint32_t); + memmove(ptr + trailer_len, ptr, header + header_len - ptr); + memcpy(ptr, trailer, trailer_len); + header_len += trailer_len; + + *mp4_header = evbuffer_new(); + evbuffer_add(*mp4_header, header, header_len); + + free(header); + free(trailer); + transcode_decode_cleanup(&ctx.decode_ctx); + transcode_encode_cleanup(&ctx.encode_ctx); + evbuffer_free(evbuf_io.evbuf); + return 0; + + error: + if (header) + DHEXDUMP(E_DBG, L_XCODE, header, header_len, "MP4 header\n"); + if (trailer) + DHEXDUMP(E_DBG, L_XCODE, trailer, trailer_len, "MP4 trailer\n"); + + free(header); + free(trailer); + transcode_decode_cleanup(&ctx.decode_ctx); + transcode_encode_cleanup(&ctx.encode_ctx); + evbuffer_free(evbuf_io.evbuf); + return -1; +} + + /* --------------------------- INPUT/OUTPUT INIT --------------------------- */ static int @@ -1051,6 +1356,19 @@ open_decoder(AVCodecContext **dec_ctx, unsigned int *stream_index, struct decode return 0; } +static void +close_input(struct decode_ctx *ctx) +{ + if (!ctx->ifmt_ctx) + return; + + avio_evbuffer_close(ctx->avio); + avcodec_free_context(&ctx->audio_stream.codec); + avcodec_free_context(&ctx->video_stream.codec); + avformat_close_input(&ctx->ifmt_ctx); + ctx->ifmt_ctx = NULL; +} + static int open_input(struct decode_ctx *ctx, const char *path, struct transcode_evbuf_io *evbuf_io, enum probe_type probe_type) { @@ -1106,7 +1424,7 @@ open_input(struct decode_ctx *ctx, const char *path, struct transcode_evbuf_io * goto out_fail; } - CHECK_NULL(L_XCODE, ctx->avio = avio_input_evbuffer_open(evbuf_io)); + CHECK_NULL(L_XCODE, ctx->avio = avio_evbuffer_open(evbuf_io, 0)); ctx->ifmt_ctx->pb = ctx->avio; ret = avformat_open_input(&ctx->ifmt_ctx, NULL, ifmt, &options); @@ -1167,25 +1485,25 @@ open_input(struct decode_ctx *ctx, const char *path, struct transcode_evbuf_io * return 0; out_fail: - avio_evbuffer_close(ctx->avio); - avcodec_free_context(&ctx->audio_stream.codec); - avcodec_free_context(&ctx->video_stream.codec); - avformat_close_input(&ctx->ifmt_ctx); - + close_input(ctx); return (ret < 0 ? ret : -1); // If we got an error code from ffmpeg then return that } static void -close_input(struct decode_ctx *ctx) +close_output(struct encode_ctx *ctx) { - avio_evbuffer_close(ctx->avio); + if (!ctx->ofmt_ctx) + return; + avcodec_free_context(&ctx->audio_stream.codec); avcodec_free_context(&ctx->video_stream.codec); - avformat_close_input(&ctx->ifmt_ctx); + avio_evbuffer_close(ctx->ofmt_ctx->pb); + avformat_free_context(ctx->ofmt_ctx); + ctx->ofmt_ctx = NULL; } static int -open_output(struct encode_ctx *ctx, struct decode_ctx *src_ctx) +open_output(struct encode_ctx *ctx, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx) { #if USE_CONST_AVFORMAT const AVOutputFormat *oformat; @@ -1193,6 +1511,8 @@ open_output(struct encode_ctx *ctx, struct decode_ctx *src_ctx) // Not const before ffmpeg 5.0 AVOutputFormat *oformat; #endif + AVDictionary *options = NULL; + struct evbuffer *header = NULL; int ret; oformat = av_guess_format(ctx->settings.format, NULL, NULL); @@ -1214,74 +1534,78 @@ open_output(struct encode_ctx *ctx, struct decode_ctx *src_ctx) ctx->ofmt_ctx->oformat = oformat; #endif - ctx->obuf = evbuffer_new(); - if (!ctx->obuf) - { - DPRINTF(E_LOG, L_XCODE, "Could not create output evbuffer\n"); - goto out_free_output; - } - - ctx->ofmt_ctx->pb = avio_output_evbuffer_open(ctx->obuf); - if (!ctx->ofmt_ctx->pb) - { - DPRINTF(E_LOG, L_XCODE, "Could not create output avio pb\n"); - goto out_free_evbuf; - } + CHECK_NULL(L_XCODE, ctx->ofmt_ctx->pb = avio_evbuffer_open(evbuf_io, 1)); + ctx->obuf = evbuf_io->evbuf; if (ctx->settings.encode_audio) { ret = stream_add(ctx, &ctx->audio_stream, ctx->settings.audio_codec); if (ret < 0) - goto out_free_streams; + goto error; } if (ctx->settings.encode_video) { ret = stream_add(ctx, &ctx->video_stream, ctx->settings.video_codec); if (ret < 0) - goto out_free_streams; + goto error; } - // Notice, this will not write WAV header (so we do that manually) - ret = avformat_write_header(ctx->ofmt_ctx, NULL); + ret = avformat_init_output(ctx->ofmt_ctx, &options); if (ret < 0) { - DPRINTF(E_LOG, L_XCODE, "Error writing header to output buffer: %s\n", err2str(ret)); - goto out_free_streams; + DPRINTF(E_LOG, L_XCODE, "Error initializing output: %s\n", err2str(ret)); + goto error; + } + else if (options) + { + DPRINTF(E_WARN, L_XCODE, "Didn't recognize all options given to avformat_init_output\n"); + av_dict_free(&options); + goto error; } + // For WAV output, both avformat_write_header() and manual wav header is required + if (!ctx->settings.without_libav_header) + { + ret = avformat_write_header(ctx->ofmt_ctx, NULL); + if (ret < 0) + { + DPRINTF(E_LOG, L_XCODE, "Error writing header to output buffer: %s\n", err2str(ret)); + goto error; + } + } if (ctx->settings.with_wav_header) { - evbuffer_add(ctx->obuf, ctx->wav_header, sizeof(ctx->wav_header)); + ret = make_wav_header(&header, ctx->settings.sample_rate, av_get_bytes_per_sample(ctx->settings.sample_format), ctx->settings.nb_channels, ctx->bytes_total); + if (ret < 0) + { + DPRINTF(E_LOG, L_XCODE, "Error creating WAV header\n"); + goto error; + } + + evbuffer_add_buffer(ctx->obuf, header); + evbuffer_free(header); + } + if (ctx->settings.with_mp4_header) + { + ret = make_mp4_header(&header, src_ctx->ifmt_ctx->url); + if (ret < 0) + { + DPRINTF(E_LOG, L_XCODE, "Error creating MP4 header\n"); + goto error; + } + + evbuffer_add_buffer(ctx->obuf, header); + evbuffer_free(header); } return 0; - out_free_streams: - avcodec_free_context(&ctx->audio_stream.codec); - avcodec_free_context(&ctx->video_stream.codec); - - avio_evbuffer_close(ctx->ofmt_ctx->pb); - out_free_evbuf: - evbuffer_free(ctx->obuf); - out_free_output: - avformat_free_context(ctx->ofmt_ctx); - + error: + close_output(ctx); return -1; } -static void -close_output(struct encode_ctx *ctx) -{ - avcodec_free_context(&ctx->audio_stream.codec); - avcodec_free_context(&ctx->video_stream.codec); - - avio_evbuffer_close(ctx->ofmt_ctx->pb); - evbuffer_free(ctx->obuf); - - avformat_free_context(ctx->ofmt_ctx); -} - static int filter_def_abuffer(struct filter_def *def, struct stream_ctx *out_stream, struct stream_ctx *in_stream, const char *deffn_arg) { @@ -1540,6 +1864,13 @@ create_filtergraph(struct stream_ctx *out_stream, struct filters *filters, size_ return -1; } +static void +close_filters(struct encode_ctx *ctx) +{ + avfilter_graph_free(&ctx->audio_stream.filter_graph); + avfilter_graph_free(&ctx->video_stream.filter_graph); +} + static int open_filters(struct encode_ctx *ctx, struct decode_ctx *src_ctx) { @@ -1576,18 +1907,10 @@ open_filters(struct encode_ctx *ctx, struct decode_ctx *src_ctx) return 0; out_fail: - avfilter_graph_free(&ctx->audio_stream.filter_graph); - avfilter_graph_free(&ctx->video_stream.filter_graph); + close_filters(ctx); return -1; } -static void -close_filters(struct encode_ctx *ctx) -{ - avfilter_graph_free(&ctx->audio_stream.filter_graph); - avfilter_graph_free(&ctx->video_stream.filter_graph); -} - /* ----------------------------- TRANSCODE API ----------------------------- */ @@ -1634,92 +1957,55 @@ transcode_decode_setup(enum transcode_profile profile, struct media_quality *qua } struct encode_ctx * -transcode_encode_setup(enum transcode_profile profile, struct media_quality *quality, struct decode_ctx *src_ctx, int width, int height) +transcode_encode_setup_with_io(enum transcode_profile profile, struct media_quality *quality, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx, int width, int height) { struct encode_ctx *ctx; - int src_bytes_per_sample; int dst_bytes_per_sample; - int channels; CHECK_NULL(L_XCODE, ctx = calloc(1, sizeof(struct encode_ctx))); CHECK_NULL(L_XCODE, ctx->filt_frame = av_frame_alloc()); CHECK_NULL(L_XCODE, ctx->encoded_pkt = av_packet_alloc()); + CHECK_NULL(L_XCODE, ctx->evbuf_io.evbuf = evbuffer_new()); + // Caller didn't specify one, so use our own + if (!evbuf_io) + evbuf_io = &ctx->evbuf_io; + + // Initialize general settings if (init_settings(&ctx->settings, profile, quality) < 0) - goto fail_free; + goto error; - ctx->settings.width = width; - ctx->settings.height = height; + if (ctx->settings.encode_audio && init_settings_from_audio(&ctx->settings, profile, src_ctx, quality) < 0) + goto error; - // Caller did not specify a sample rate -> use same as source - if (!ctx->settings.sample_rate && ctx->settings.encode_audio) - { - ctx->settings.sample_rate = src_ctx->audio_stream.codec->sample_rate; - } - - // Caller did not specify a sample format -> determine from source - if (!ctx->settings.sample_format && ctx->settings.encode_audio) - { - src_bytes_per_sample = av_get_bytes_per_sample(src_ctx->audio_stream.codec->sample_fmt); - if (src_bytes_per_sample == 4) - { - ctx->settings.sample_format = AV_SAMPLE_FMT_S32; - ctx->settings.audio_codec = AV_CODEC_ID_PCM_S32LE; - ctx->settings.format = "s32le"; - } - else - { - ctx->settings.sample_format = AV_SAMPLE_FMT_S16; - ctx->settings.audio_codec = AV_CODEC_ID_PCM_S16LE; - ctx->settings.format = "s16le"; - } - } - -#if USE_CH_LAYOUT - // Caller did not specify channels -> use same as source - if (!av_channel_layout_check(&ctx->settings.channel_layout) && ctx->settings.encode_audio) - { - av_channel_layout_copy(&ctx->settings.channel_layout, &src_ctx->audio_stream.codec->ch_layout); - } - - channels = ctx->settings.channel_layout.nb_channels; -#else - // Caller did not specify channels -> use same as source - if (ctx->settings.channels == 0 && ctx->settings.encode_audio) - { - ctx->settings.channels = src_ctx->audio_stream.codec->channels; - ctx->settings.channel_layout = src_ctx->audio_stream.codec->channel_layout; - } - - channels = ctx->settings.channels; -#endif + if (ctx->settings.encode_video && init_settings_from_video(&ctx->settings, profile, src_ctx, width, height) < 0) + goto error; dst_bytes_per_sample = av_get_bytes_per_sample(ctx->settings.sample_format); + ctx->bytes_total = size_estimate(profile, ctx->settings.bit_rate, ctx->settings.sample_rate, dst_bytes_per_sample, ctx->settings.nb_channels, src_ctx->len_ms); - ctx->bytes_total = size_estimate(profile, ctx->settings.bit_rate, ctx->settings.sample_rate, dst_bytes_per_sample, channels, src_ctx->len_ms); - - if (ctx->settings.with_wav_header) - make_wav_header(ctx->wav_header, ctx->settings.sample_rate, dst_bytes_per_sample, channels, ctx->bytes_total); if (ctx->settings.with_icy && src_ctx->data_kind == DATA_KIND_HTTP) - ctx->icy_interval = METADATA_ICY_INTERVAL * channels * dst_bytes_per_sample * ctx->settings.sample_rate; + ctx->icy_interval = METADATA_ICY_INTERVAL * ctx->settings.nb_channels * dst_bytes_per_sample * ctx->settings.sample_rate; - if (open_output(ctx, src_ctx) < 0) - goto fail_free; + if (open_output(ctx, evbuf_io, src_ctx) < 0) + goto error; if (open_filters(ctx, src_ctx) < 0) - goto fail_close; + goto error; return ctx; - fail_close: - close_output(ctx); - fail_free: - av_packet_free(&ctx->encoded_pkt); - av_frame_free(&ctx->filt_frame); - free(ctx); + error: + transcode_encode_cleanup(&ctx); return NULL; } +struct encode_ctx * +transcode_encode_setup(enum transcode_profile profile, struct media_quality *quality, struct decode_ctx *src_ctx, int width, int height) +{ + return transcode_encode_setup_with_io(profile, quality, NULL, src_ctx, width, height); +} + struct transcode_ctx * transcode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, uint32_t len_ms) { @@ -1814,6 +2100,7 @@ transcode_needed(const char *user_agent, const char *client_codecs, char *file_c const char *prefer_format; cfg_t *lib; bool force_xcode; + bool supports_alac; bool supports_mpeg; bool supports_wav; int count; @@ -1872,6 +2159,7 @@ transcode_needed(const char *user_agent, const char *client_codecs, char *file_c if (!force_xcode && strstr(client_codecs, file_codectype)) return XCODE_NONE; + supports_alac = strstr(client_codecs, "alac") || strstr(client_codecs, "mp4a"); supports_mpeg = strstr(client_codecs, "mpeg") && avcodec_find_encoder(AV_CODEC_ID_MP3); supports_wav = strstr(client_codecs, "wav"); @@ -1880,18 +2168,22 @@ transcode_needed(const char *user_agent, const char *client_codecs, char *file_c { if (strcmp(prefer_format, "wav") == 0 && supports_wav) return XCODE_WAV; - else if (strcmp(prefer_format, "mpeg") == 0 && supports_mpeg) + if (strcmp(prefer_format, "mpeg") == 0 && supports_mpeg) return XCODE_MP3; + if (strcmp(prefer_format, "alac") == 0 && supports_alac) + return XCODE_MP4_ALAC; } // This order determines the default if user didn't configure a preference. // The lossless formats are given highest preference. if (supports_wav) return XCODE_WAV; - else if (supports_mpeg) + if (supports_mpeg) return XCODE_MP3; - else - return XCODE_UNKNOWN; + if (supports_alac) + return XCODE_MP4_ALAC; + + return XCODE_UNKNOWN; } @@ -1920,6 +2212,7 @@ transcode_encode_cleanup(struct encode_ctx **ctx) close_filters(*ctx); close_output(*ctx); + evbuffer_free((*ctx)->evbuf_io.evbuf); av_packet_free(&(*ctx)->encoded_pkt); av_frame_free(&(*ctx)->filt_frame); free(*ctx); @@ -2312,6 +2605,17 @@ transcode_metadata_strings_set(struct transcode_metadata_string *s, enum transco snprintf(s->file_size, sizeof(s->file_size), "%d", (int)bytes); break; + case XCODE_MP4_ALAC: + s->type = "m4a"; + s->codectype = "alac"; + s->description = "Apple Lossless audio file"; + + snprintf(s->bitrate, sizeof(s->bitrate), "%d", 8 * STOB(q->sample_rate, q->bits_per_sample, q->channels) / 1000); // 44100/16/2 -> 1411 + + bytes = size_estimate(profile, q->bit_rate, q->sample_rate, q->bits_per_sample / 8, q->channels, len_ms); + snprintf(s->file_size, sizeof(s->file_size), "%d", (int)bytes); + break; + default: DPRINTF(E_WARN, L_XCODE, "transcode_metadata_strings_set() called with unknown profile %d\n", profile); } diff --git a/src/transcode.h b/src/transcode.h index d0550dc1..f72b7a4e 100644 --- a/src/transcode.h +++ b/src/transcode.h @@ -23,10 +23,14 @@ enum transcode_profile XCODE_PCM32, // Transcodes the best audio stream to MP3 XCODE_MP3, - // Transcodes the best audio stream to OPUS + // Transcodes the best audio stream to raw OPUS (no container) XCODE_OPUS, - // Transcodes the best audio stream to ALAC + // Transcodes the best audio stream to raw ALAC (no container) XCODE_ALAC, + // Transcodes the best audio stream to ALAC in a MP4 container + XCODE_MP4_ALAC, + // Produces just the header for a MP4 container with ALAC + XCODE_MP4_ALAC_HEADER, // Transcodes the best audio stream from OGG XCODE_OGG, // Transcodes the best video stream to JPEG/PNG/VP8 @@ -79,6 +83,9 @@ transcode_decode_setup(enum transcode_profile profile, struct media_quality *qua struct encode_ctx * transcode_encode_setup(enum transcode_profile profile, struct media_quality *quality, struct decode_ctx *src_ctx, int width, int height); +struct encode_ctx * +transcode_encode_setup_with_io(enum transcode_profile profile, struct media_quality *quality, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx, int width, int height); + struct transcode_ctx * transcode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, uint32_t len_ms); @@ -182,9 +189,15 @@ transcode_encode_query(struct encode_ctx *ctx, const char *query); struct http_icy_metadata * transcode_metadata(struct transcode_ctx *ctx, int *changed); -// When transcoding, we are in essence serving a different source file than the -// original to the client. So we can't serve some of the file metadata from the -// filescanner. This function creates strings to be used for override. +/* When transcoding, we are in essence serving a different source file than the + * original to the client. So we can't serve some of the file metadata from the + * filescanner. This function creates strings to be used for override. + * + * @out s Structure with (non-allocated) strings + * @in profile Transcoding profile + * @in q Transcoding quality + * @in len_ms Length of source track + */ void transcode_metadata_strings_set(struct transcode_metadata_string *s, enum transcode_profile profile, struct media_quality *q, uint32_t len_ms); From 725419d4ac876d68030cf954787f6e137aae7d1c Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Tue, 19 Dec 2023 22:36:45 +0100 Subject: [PATCH 02/12] [xcode] Do without _GNU_SOURCE, MacOS doesn't like it --- src/transcode.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/transcode.c b/src/transcode.c index 632df7df..45887c2a 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -20,8 +20,6 @@ # include #endif -#define _GNU_SOURCE // For memmem() - #include #include #include From 9dbec4b99e47b9ce071facec1ea1b6d30c2ca7b6 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Tue, 19 Dec 2023 22:46:25 +0100 Subject: [PATCH 03/12] [misc] Move endianess utils from rtp_common.h to misc.h So that transcode.c can also use them. --- src/misc.h | 12 ++++++++++++ src/outputs/cast.c | 11 ++--------- src/outputs/rtp_common.h | 12 ------------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/misc.h b/src/misc.h index 520fe896..e3d1b7d9 100644 --- a/src/misc.h +++ b/src/misc.h @@ -59,6 +59,18 @@ net_is_http_or_https(const char *url); /* ----------------------- Conversion/hashing/sanitizers -------------------- */ +#ifdef HAVE_ENDIAN_H +# include +#elif defined(HAVE_SYS_ENDIAN_H) +# include +#elif defined(HAVE_LIBKERN_OSBYTEORDER_H) +#include +#define htobe16(x) OSSwapHostToBigInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#endif + // Samples to bytes, bytes to samples #define STOB(s, bits, c) ((s) * (c) * (bits) / 8) #define BTOS(b, bits, c) ((b) / ((c) * (bits) / 8)) diff --git a/src/outputs/cast.c b/src/outputs/cast.c index 7963d0be..d051b1d0 100644 --- a/src/outputs/cast.c +++ b/src/outputs/cast.c @@ -32,15 +32,7 @@ #include #include #include -#ifdef HAVE_ENDIAN_H -# include -#elif defined(HAVE_SYS_ENDIAN_H) -# include -#elif defined(HAVE_LIBKERN_OSBYTEORDER_H) -#include -#define htobe32(x) OSSwapHostToBigInt32(x) -#define be32toh(x) OSSwapBigToHostInt32(x) -#endif + #include #include #include @@ -55,6 +47,7 @@ #include "outputs.h" #include "db.h" #include "artwork.h" +#include "misc.h" #ifdef HAVE_PROTOBUF_OLD #include "cast_channel.v0.pb-c.h" diff --git a/src/outputs/rtp_common.h b/src/outputs/rtp_common.h index d6ced5c7..0900157c 100644 --- a/src/outputs/rtp_common.h +++ b/src/outputs/rtp_common.h @@ -5,18 +5,6 @@ #include #include -#ifdef HAVE_ENDIAN_H -# include -#elif defined(HAVE_SYS_ENDIAN_H) -# include -#elif defined(HAVE_LIBKERN_OSBYTEORDER_H) -#include -#define htobe16(x) OSSwapHostToBigInt16(x) -#define be16toh(x) OSSwapBigToHostInt16(x) -#define htobe32(x) OSSwapHostToBigInt32(x) -#define be32toh(x) OSSwapBigToHostInt32(x) -#endif - struct rtcp_timestamp { uint32_t pos; From 4a08644806199b2f54c4417b16a132fa15792ce7 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Mon, 25 Dec 2023 22:29:32 +0100 Subject: [PATCH 04/12] [xcode] Change the signature of transcode_setup/transcode_en/decode_setup() Should make it easier to add/remove parameters without changing all calls to the functions throughout the code. Also adds an interface through which to call make_mp4_header(). --- src/artwork.c | 21 +++++--- src/httpd.c | 16 ++++-- src/inputs/file.c | 4 +- src/inputs/http.c | 5 +- src/inputs/spotify_librespotc.c | 15 +++--- src/outputs.c | 15 +++--- src/outputs/airplay.c | 10 ++-- src/outputs/cast.c | 10 ++-- src/outputs/raop.c | 10 ++-- src/outputs/streaming.c | 14 ++--- src/transcode.c | 92 ++++++++++++++++++++------------- src/transcode.h | 44 +++++++++++++--- 12 files changed, 165 insertions(+), 91 deletions(-) diff --git a/src/artwork.c b/src/artwork.c index 7d51986c..697228f7 100644 --- a/src/artwork.c +++ b/src/artwork.c @@ -604,6 +604,8 @@ size_calculate(int *dst_w, int *dst_h, int src_w, int src_h, int max_w, int max_ static int artwork_get(struct evbuffer *evbuf, char *path, struct evbuffer *in_buf, bool is_embedded, enum data_kind data_kind, struct artwork_req_params req_params) { + struct transcode_decode_setup_args xcode_decode_args = { .profile = XCODE_JPEG }; // Covers XCODE_PNG too + struct transcode_encode_setup_args xcode_encode_args = { 0 }; struct decode_ctx *xcode_decode = NULL; struct encode_ctx *xcode_encode = NULL; struct transcode_evbuf_io xcode_evbuf_io = { 0 }; @@ -637,13 +639,16 @@ artwork_get(struct evbuffer *evbuf, char *path, struct evbuffer *in_buf, bool is } xcode_evbuf_io.evbuf = xcode_buf; - xcode_decode = transcode_decode_setup(XCODE_JPEG, NULL, data_kind, NULL, &xcode_evbuf_io, 0); // Covers XCODE_PNG too + xcode_decode_args.evbuf_io = &xcode_evbuf_io; + xcode_decode_args.is_http = (data_kind == DATA_KIND_HTTP); } else { - xcode_decode = transcode_decode_setup(XCODE_JPEG, NULL, data_kind, path, NULL, 0); // Covers XCODE_PNG too + xcode_decode_args.path = path; + xcode_decode_args.is_http = (data_kind == DATA_KIND_HTTP); } + xcode_decode = transcode_decode_setup(xcode_decode_args); if (!xcode_decode) { if (path) @@ -702,15 +707,19 @@ artwork_get(struct evbuffer *evbuf, char *path, struct evbuffer *in_buf, bool is goto out; } + xcode_encode_args.src_ctx = xcode_decode; + xcode_encode_args.width = dst_width; + xcode_encode_args.height = dst_height; if (dst_format == ART_FMT_JPEG) - xcode_encode = transcode_encode_setup(XCODE_JPEG, NULL, xcode_decode, dst_width, dst_height); + xcode_encode_args.profile = XCODE_JPEG; else if (dst_format == ART_FMT_PNG) - xcode_encode = transcode_encode_setup(XCODE_PNG, NULL, xcode_decode, dst_width, dst_height); + xcode_encode_args.profile = XCODE_PNG; else if (dst_format == ART_FMT_VP8) - xcode_encode = transcode_encode_setup(XCODE_VP8, NULL, xcode_decode, dst_width, dst_height); + xcode_encode_args.profile = XCODE_VP8; else - xcode_encode = transcode_encode_setup(XCODE_JPEG, NULL, xcode_decode, dst_width, dst_height); + xcode_encode_args.profile = XCODE_JPEG; + xcode_encode = transcode_encode_setup(xcode_encode_args); if (!xcode_encode) { if (path) diff --git a/src/httpd.c b/src/httpd.c index 65a50720..d2f3fb96 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -673,18 +673,28 @@ static struct stream_ctx * stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile, struct httpd_request *hreq, int64_t offset, int64_t end_offset, event_callback_fn stream_cb) { + struct transcode_decode_setup_args decode_args = { 0 }; + struct transcode_encode_setup_args encode_args = { 0 }; struct media_quality quality = { 0 }; struct stream_ctx *st; + // We use source sample rate etc, but for MP3 we must set a bit rate + quality.bit_rate = 1000 * cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); + st = stream_new(mfi, hreq, stream_cb); if (!st) { goto error; } - // We use source sample rate etc, but for MP3 we must set a bit rate - quality.bit_rate = cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); - st->xcode = transcode_setup(profile, &quality, mfi->data_kind, mfi->path, mfi->song_length); + decode_args.profile = profile; + decode_args.is_http = (mfi->data_kind == DATA_KIND_HTTP); + decode_args.path = mfi->path; + decode_args.len_ms = mfi->song_length; + encode_args.profile = profile; + encode_args.quality = &quality; + + st->xcode = transcode_setup(decode_args, encode_args); if (!st->xcode) { DPRINTF(E_WARN, L_HTTPD, "Transcoding setup failed, aborting streaming\n"); diff --git a/src/inputs/file.c b/src/inputs/file.c index ee3ed56c..68fe4c30 100644 --- a/src/inputs/file.c +++ b/src/inputs/file.c @@ -36,9 +36,11 @@ static int setup(struct input_source *source) { + struct transcode_decode_setup_args decode_args = { .profile = XCODE_PCM_NATIVE, .path = source->path, .len_ms = source->len_ms }; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_PCM_NATIVE, }; struct transcode_ctx *ctx; - ctx = transcode_setup(XCODE_PCM_NATIVE, NULL, source->data_kind, source->path, source->len_ms); + ctx = transcode_setup(decode_args, encode_args); if (!ctx) return -1; diff --git a/src/inputs/http.c b/src/inputs/http.c index 75d46e8d..5a671b54 100644 --- a/src/inputs/http.c +++ b/src/inputs/http.c @@ -295,6 +295,8 @@ metadata_prepare(struct input_source *source) static int setup(struct input_source *source) { + struct transcode_decode_setup_args decode_args = { .profile = XCODE_PCM_NATIVE, .is_http = true, .len_ms = source->len_ms }; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_PCM_NATIVE, }; struct transcode_ctx *ctx; char *url; @@ -303,8 +305,9 @@ setup(struct input_source *source) free(source->path); source->path = url; + decode_args.path = url; - ctx = transcode_setup(XCODE_PCM_NATIVE, NULL, source->data_kind, source->path, source->len_ms); + ctx = transcode_setup(decode_args, encode_args); if (!ctx) return -1; diff --git a/src/inputs/spotify_librespotc.c b/src/inputs/spotify_librespotc.c index e1e2bfb4..d973bf8a 100644 --- a/src/inputs/spotify_librespotc.c +++ b/src/inputs/spotify_librespotc.c @@ -391,21 +391,18 @@ download_seek(void *arg, int64_t offset, enum transcode_seek_type type) static int download_xcode_setup(struct download_ctx *download) { - struct transcode_ctx *xcode; + struct transcode_decode_setup_args decode_args = { .profile = XCODE_OGG, .len_ms = download->len_ms }; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_PCM16, }; struct transcode_evbuf_io xcode_evbuf_io = { 0 }; - - CHECK_NULL(L_SPOTIFY, xcode = malloc(sizeof(struct transcode_ctx))); + struct transcode_ctx *xcode; xcode_evbuf_io.evbuf = download->read_buf; xcode_evbuf_io.seekfn = download_seek; xcode_evbuf_io.seekfn_arg = download; + decode_args.evbuf_io = &xcode_evbuf_io; - xcode->decode_ctx = transcode_decode_setup(XCODE_OGG, NULL, DATA_KIND_SPOTIFY, NULL, &xcode_evbuf_io, download->len_ms); - if (!xcode->decode_ctx) - goto error; - - xcode->encode_ctx = transcode_encode_setup(XCODE_PCM16, NULL, xcode->decode_ctx, 0, 0); - if (!xcode->encode_ctx) + xcode = transcode_setup(decode_args, encode_args); + if (!xcode) goto error; download->xcode = xcode; diff --git a/src/outputs.c b/src/outputs.c index d6469a4d..a4aa9425 100644 --- a/src/outputs.c +++ b/src/outputs.c @@ -280,8 +280,8 @@ quality_to_xcode(struct media_quality *quality) static int encoding_reset(struct media_quality *quality) { + struct transcode_encode_setup_args encode_args = { 0 }; struct output_quality_subscription *subscription; - struct decode_ctx *decode_ctx; enum transcode_profile profile; int i; @@ -293,8 +293,8 @@ encoding_reset(struct media_quality *quality) return -1; } - decode_ctx = transcode_decode_setup_raw(profile, quality); - if (!decode_ctx) + encode_args.src_ctx = transcode_decode_setup_raw(profile, quality); + if (!encode_args.src_ctx) { DPRINTF(E_LOG, L_PLAYER, "Could not create subscription decoding context (profile %d)\n", profile); return -1; @@ -309,15 +309,16 @@ encoding_reset(struct media_quality *quality) if (quality_is_equal(quality, &subscription->quality)) continue; // No resampling required - profile = quality_to_xcode(&subscription->quality); - if (profile != XCODE_UNKNOWN) - subscription->encode_ctx = transcode_encode_setup(profile, &subscription->quality, decode_ctx, 0, 0); + encode_args.profile = quality_to_xcode(&subscription->quality); + encode_args.quality = &subscription->quality; + if (encode_args.profile != XCODE_UNKNOWN) + subscription->encode_ctx = transcode_encode_setup(encode_args); else DPRINTF(E_LOG, L_PLAYER, "Could not setup resampling to %d/%d/%d for output\n", subscription->quality.sample_rate, subscription->quality.bits_per_sample, subscription->quality.channels); } - transcode_decode_cleanup(&decode_ctx); + transcode_decode_cleanup(&encode_args.src_ctx); return 0; } diff --git a/src/outputs/airplay.c b/src/outputs/airplay.c index afd53b50..7e6b3cf7 100644 --- a/src/outputs/airplay.c +++ b/src/outputs/airplay.c @@ -1120,7 +1120,7 @@ static struct airplay_master_session * master_session_make(struct media_quality *quality) { struct airplay_master_session *rms; - struct decode_ctx *decode_ctx; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_ALAC, .quality = quality }; int ret; // First check if we already have a suitable session @@ -1146,15 +1146,15 @@ master_session_make(struct media_quality *quality) goto error; } - decode_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); - if (!decode_ctx) + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); + if (!encode_args.src_ctx) { DPRINTF(E_LOG, L_AIRPLAY, "Could not create decoding context\n"); goto error; } - rms->encode_ctx = transcode_encode_setup(XCODE_ALAC, quality, decode_ctx, 0, 0); - transcode_decode_cleanup(&decode_ctx); + rms->encode_ctx = transcode_encode_setup(encode_args); + transcode_decode_cleanup(&encode_args.src_ctx); if (!rms->encode_ctx) { DPRINTF(E_LOG, L_AIRPLAY, "Will not be able to stream AirPlay 2, ffmpeg has no ALAC encoder\n"); diff --git a/src/outputs/cast.c b/src/outputs/cast.c index d051b1d0..b6cc5ffb 100644 --- a/src/outputs/cast.c +++ b/src/outputs/cast.c @@ -2362,7 +2362,7 @@ cast_metadata_send(struct output_metadata *metadata) static int cast_init(void) { - struct decode_ctx *decode_ctx; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_OPUS, .quality = &cast_quality_default }; int i; int ret; @@ -2386,15 +2386,15 @@ cast_init(void) return -1; } - decode_ctx = transcode_decode_setup_raw(XCODE_PCM16, &cast_quality_default); - if (!decode_ctx) + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM16, &cast_quality_default); + if (!encode_args.src_ctx) { DPRINTF(E_LOG, L_CAST, "Could not create decoding context\n"); goto out_tls_deinit; } - cast_encode_ctx = transcode_encode_setup(XCODE_OPUS, &cast_quality_default, decode_ctx, 0, 0); - transcode_decode_cleanup(&decode_ctx); + cast_encode_ctx = transcode_encode_setup(encode_args); + transcode_decode_cleanup(&encode_args.src_ctx); if (!cast_encode_ctx) { DPRINTF(E_LOG, L_CAST, "Will not be able to stream Chromecast, libav does not support Opus encoding\n"); diff --git a/src/outputs/raop.c b/src/outputs/raop.c index ec0a61e6..20436aaa 100644 --- a/src/outputs/raop.c +++ b/src/outputs/raop.c @@ -1855,7 +1855,7 @@ static struct raop_master_session * master_session_make(struct media_quality *quality, bool encrypt) { struct raop_master_session *rms; - struct decode_ctx *decode_ctx; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_ALAC, .quality = quality }; int ret; // First check if we already have a suitable session @@ -1883,15 +1883,15 @@ master_session_make(struct media_quality *quality, bool encrypt) return NULL; } - decode_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); - if (!decode_ctx) + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); + if (!encode_args.src_ctx) { DPRINTF(E_LOG, L_RAOP, "Could not create decoding context\n"); goto error; } - rms->encode_ctx = transcode_encode_setup(XCODE_ALAC, quality, decode_ctx, 0, 0); - transcode_decode_cleanup(&decode_ctx); + rms->encode_ctx = transcode_encode_setup(encode_args); + transcode_decode_cleanup(&encode_args.src_ctx); if (!rms->encode_ctx) { DPRINTF(E_LOG, L_RAOP, "Will not be able to stream AirPlay 2, ffmpeg has no ALAC encoder\n"); diff --git a/src/outputs/streaming.c b/src/outputs/streaming.c index 68885210..ecb5314b 100644 --- a/src/outputs/streaming.c +++ b/src/outputs/streaming.c @@ -115,17 +115,17 @@ extern struct event_base *evbase_player; static struct encode_ctx * encoder_setup(enum player_format format, struct media_quality *quality) { - struct decode_ctx *decode_ctx = NULL; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_MP3, .quality = quality }; struct encode_ctx *encode_ctx = NULL; if (quality->bits_per_sample == 16) - decode_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM16, quality); else if (quality->bits_per_sample == 24) - decode_ctx = transcode_decode_setup_raw(XCODE_PCM24, quality); + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM24, quality); else if (quality->bits_per_sample == 32) - decode_ctx = transcode_decode_setup_raw(XCODE_PCM32, quality); + encode_args.src_ctx = transcode_decode_setup_raw(XCODE_PCM32, quality); - if (!decode_ctx) + if (!encode_args.src_ctx) { DPRINTF(E_LOG, L_STREAMING, "Error setting up decoder for quality sr %d, bps %d, ch %d, cannot encode\n", quality->sample_rate, quality->bits_per_sample, quality->channels); @@ -133,7 +133,7 @@ encoder_setup(enum player_format format, struct media_quality *quality) } if (format == PLAYER_FORMAT_MP3) - encode_ctx = transcode_encode_setup(XCODE_MP3, quality, decode_ctx, 0, 0); + encode_ctx = transcode_encode_setup(encode_args); if (!encode_ctx) { @@ -143,7 +143,7 @@ encoder_setup(enum player_format format, struct media_quality *quality) } out: - transcode_decode_cleanup(&decode_ctx); + transcode_decode_cleanup(&encode_args.src_ctx); return encode_ctx; } diff --git a/src/transcode.c b/src/transcode.c index 45887c2a..abcfb0d8 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -38,7 +38,6 @@ #include "logger.h" #include "conffile.h" -#include "db.h" #include "misc.h" #include "transcode.h" @@ -151,8 +150,8 @@ struct decode_ctx // Source duration in ms as provided by caller uint32_t len_ms; - // Data kind (used to determine if ICY metadata is relevant to look for) - enum data_kind data_kind; + // Used to determine if ICY metadata is relevant to look for + bool is_http; // Set to true if we just seeked bool resume; @@ -1202,6 +1201,8 @@ mp4_header_trailer_from_evbuf(uint8_t **header, size_t *header_len, uint8_t **tr static int make_mp4_header(struct evbuffer **mp4_header, const char *url) { + struct transcode_decode_setup_args decode_args = { .profile = XCODE_MP4_ALAC_HEADER }; + struct transcode_encode_setup_args encode_args = { .profile = XCODE_MP4_ALAC_HEADER }; struct transcode_ctx ctx = { 0 }; struct transcode_evbuf_io evbuf_io = { 0 }; uint8_t free_tag[4] = { 'f', 'r', 'e', 'e' }; @@ -1220,11 +1221,14 @@ make_mp4_header(struct evbuffer **mp4_header, const char *url) evbuf_io.seekfn = dummy_seek; evbuf_io.seekfn_arg = &ctx; - ctx.decode_ctx = transcode_decode_setup(XCODE_MP4_ALAC_HEADER, NULL, DATA_KIND_FILE, url, NULL, -1); + decode_args.path = url; + ctx.decode_ctx = transcode_decode_setup(decode_args); if (!ctx.decode_ctx) goto error; - ctx.encode_ctx = transcode_encode_setup_with_io(XCODE_MP4_ALAC_HEADER, NULL, &evbuf_io, ctx.decode_ctx, 0, 0); + encode_args.evbuf_io = &evbuf_io; + encode_args.src_ctx = ctx.decode_ctx; + ctx.encode_ctx = transcode_encode_setup(encode_args); if (!ctx.encode_ctx) goto error; @@ -1394,7 +1398,7 @@ open_input(struct decode_ctx *ctx, const char *path, struct transcode_evbuf_io * ctx->ifmt_ctx->format_probesize = 65536; } - if (ctx->data_kind == DATA_KIND_HTTP) + if (ctx->is_http) { av_dict_set(&options, "icy", "1", 0); @@ -1501,7 +1505,7 @@ close_output(struct encode_ctx *ctx) } static int -open_output(struct encode_ctx *ctx, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx) +open_output(struct encode_ctx *ctx, struct transcode_evbuf_io *evbuf_io, struct evbuffer *prepared_header, struct decode_ctx *src_ctx) { #if USE_CONST_AVFORMAT const AVOutputFormat *oformat; @@ -1584,7 +1588,12 @@ open_output(struct encode_ctx *ctx, struct transcode_evbuf_io *evbuf_io, struct evbuffer_add_buffer(ctx->obuf, header); evbuffer_free(header); } - if (ctx->settings.with_mp4_header) + + if (ctx->settings.with_mp4_header && prepared_header) + { + evbuffer_add_buffer(ctx->obuf, prepared_header); + } + else if (ctx->settings.with_mp4_header) { ret = make_mp4_header(&header, src_ctx->ifmt_ctx->url); if (ret < 0) @@ -1915,7 +1924,7 @@ open_filters(struct encode_ctx *ctx, struct decode_ctx *src_ctx) /* Setup */ struct decode_ctx * -transcode_decode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, struct transcode_evbuf_io *evbuf_io, uint32_t len_ms) +transcode_decode_setup(struct transcode_decode_setup_args args) { struct decode_ctx *ctx; int ret; @@ -1924,23 +1933,24 @@ transcode_decode_setup(enum transcode_profile profile, struct media_quality *qua CHECK_NULL(L_XCODE, ctx->decoded_frame = av_frame_alloc()); CHECK_NULL(L_XCODE, ctx->packet = av_packet_alloc()); - ctx->len_ms = len_ms; - ctx->data_kind = data_kind; + ctx->len_ms = args.len_ms; - ret = init_settings(&ctx->settings, profile, quality); + ret = init_settings(&ctx->settings, args.profile, args.quality); if (ret < 0) goto fail_free; - if (data_kind == DATA_KIND_HTTP) + if (args.is_http) { - ret = open_input(ctx, path, evbuf_io, PROBE_TYPE_QUICK); + ctx->is_http = true; + + ret = open_input(ctx, args.path, args.evbuf_io, PROBE_TYPE_QUICK); // Retry with a default, slower probe size if (ret == AVERROR_STREAM_NOT_FOUND) - ret = open_input(ctx, path, evbuf_io, PROBE_TYPE_DEFAULT); + ret = open_input(ctx, args.path, args.evbuf_io, PROBE_TYPE_DEFAULT); } else - ret = open_input(ctx, path, evbuf_io, PROBE_TYPE_DEFAULT); + ret = open_input(ctx, args.path, args.evbuf_io, PROBE_TYPE_DEFAULT); if (ret < 0) goto fail_free; @@ -1955,7 +1965,7 @@ transcode_decode_setup(enum transcode_profile profile, struct media_quality *qua } struct encode_ctx * -transcode_encode_setup_with_io(enum transcode_profile profile, struct media_quality *quality, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx, int width, int height) +transcode_encode_setup(struct transcode_encode_setup_args args) { struct encode_ctx *ctx; int dst_bytes_per_sample; @@ -1966,29 +1976,29 @@ transcode_encode_setup_with_io(enum transcode_profile profile, struct media_qual CHECK_NULL(L_XCODE, ctx->evbuf_io.evbuf = evbuffer_new()); // Caller didn't specify one, so use our own - if (!evbuf_io) - evbuf_io = &ctx->evbuf_io; + if (!args.evbuf_io) + args.evbuf_io = &ctx->evbuf_io; // Initialize general settings - if (init_settings(&ctx->settings, profile, quality) < 0) + if (init_settings(&ctx->settings, args.profile, args.quality) < 0) goto error; - if (ctx->settings.encode_audio && init_settings_from_audio(&ctx->settings, profile, src_ctx, quality) < 0) + if (ctx->settings.encode_audio && init_settings_from_audio(&ctx->settings, args.profile, args.src_ctx, args.quality) < 0) goto error; - if (ctx->settings.encode_video && init_settings_from_video(&ctx->settings, profile, src_ctx, width, height) < 0) + if (ctx->settings.encode_video && init_settings_from_video(&ctx->settings, args.profile, args.src_ctx, args.width, args.height) < 0) goto error; dst_bytes_per_sample = av_get_bytes_per_sample(ctx->settings.sample_format); - ctx->bytes_total = size_estimate(profile, ctx->settings.bit_rate, ctx->settings.sample_rate, dst_bytes_per_sample, ctx->settings.nb_channels, src_ctx->len_ms); + ctx->bytes_total = size_estimate(args.profile, ctx->settings.bit_rate, ctx->settings.sample_rate, dst_bytes_per_sample, ctx->settings.nb_channels, args.src_ctx->len_ms); - if (ctx->settings.with_icy && src_ctx->data_kind == DATA_KIND_HTTP) + if (ctx->settings.with_icy && args.src_ctx->is_http) ctx->icy_interval = METADATA_ICY_INTERVAL * ctx->settings.nb_channels * dst_bytes_per_sample * ctx->settings.sample_rate; - if (open_output(ctx, evbuf_io, src_ctx) < 0) + if (open_output(ctx, args.evbuf_io, args.prepared_header, args.src_ctx) < 0) goto error; - if (open_filters(ctx, src_ctx) < 0) + if (open_filters(ctx, args.src_ctx) < 0) goto error; return ctx; @@ -1998,27 +2008,22 @@ transcode_encode_setup_with_io(enum transcode_profile profile, struct media_qual return NULL; } -struct encode_ctx * -transcode_encode_setup(enum transcode_profile profile, struct media_quality *quality, struct decode_ctx *src_ctx, int width, int height) -{ - return transcode_encode_setup_with_io(profile, quality, NULL, src_ctx, width, height); -} - struct transcode_ctx * -transcode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, uint32_t len_ms) +transcode_setup(struct transcode_decode_setup_args decode_args, struct transcode_encode_setup_args encode_args) { struct transcode_ctx *ctx; CHECK_NULL(L_XCODE, ctx = calloc(1, sizeof(struct transcode_ctx))); - ctx->decode_ctx = transcode_decode_setup(profile, quality, data_kind, path, NULL, len_ms); + ctx->decode_ctx = transcode_decode_setup(decode_args); if (!ctx->decode_ctx) { free(ctx); return NULL; } - ctx->encode_ctx = transcode_encode_setup(profile, quality, ctx->decode_ctx, 0, 0); + encode_args.src_ctx = ctx->decode_ctx; + ctx->encode_ctx = transcode_encode_setup(encode_args); if (!ctx->encode_ctx) { transcode_decode_cleanup(&ctx->decode_ctx); @@ -2618,3 +2623,20 @@ transcode_metadata_strings_set(struct transcode_metadata_string *s, enum transco DPRINTF(E_WARN, L_XCODE, "transcode_metadata_strings_set() called with unknown profile %d\n", profile); } } + +int +transcode_create_header(struct evbuffer **header, enum transcode_profile profile, const char *path) +{ + int ret; + + switch (profile) + { + case XCODE_MP4_ALAC: + ret = make_mp4_header(header, path); + break; + default: + ret = -1; + } + + return ret; +} diff --git a/src/transcode.h b/src/transcode.h index f72b7a4e..c9a7f372 100644 --- a/src/transcode.h +++ b/src/transcode.h @@ -3,7 +3,6 @@ #define __TRANSCODE_H__ #include -#include "db.h" #include "http.h" #include "misc.h" @@ -66,6 +65,29 @@ struct transcode_evbuf_io void *seekfn_arg; }; +struct transcode_decode_setup_args +{ + enum transcode_profile profile; + struct media_quality *quality; + bool is_http; + uint32_t len_ms; + + // Source must be either of these + const char *path; + struct transcode_evbuf_io *evbuf_io; +}; + +struct transcode_encode_setup_args +{ + enum transcode_profile profile; + struct media_quality *quality; + struct decode_ctx *src_ctx; + struct transcode_evbuf_io *evbuf_io; + struct evbuffer *prepared_header; + int width; + int height; +}; + struct transcode_metadata_string { char *type; @@ -78,16 +100,13 @@ struct transcode_metadata_string // Setting up struct decode_ctx * -transcode_decode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, struct transcode_evbuf_io *evbuf_io, uint32_t len_ms); +transcode_decode_setup(struct transcode_decode_setup_args args); struct encode_ctx * -transcode_encode_setup(enum transcode_profile profile, struct media_quality *quality, struct decode_ctx *src_ctx, int width, int height); - -struct encode_ctx * -transcode_encode_setup_with_io(enum transcode_profile profile, struct media_quality *quality, struct transcode_evbuf_io *evbuf_io, struct decode_ctx *src_ctx, int width, int height); +transcode_encode_setup(struct transcode_encode_setup_args args); struct transcode_ctx * -transcode_setup(enum transcode_profile profile, struct media_quality *quality, enum data_kind data_kind, const char *path, uint32_t len_ms); +transcode_setup(struct transcode_decode_setup_args decode_args, struct transcode_encode_setup_args encode_args); struct decode_ctx * transcode_decode_setup_raw(enum transcode_profile profile, struct media_quality *quality); @@ -201,4 +220,15 @@ transcode_metadata(struct transcode_ctx *ctx, int *changed); void transcode_metadata_strings_set(struct transcode_metadata_string *s, enum transcode_profile profile, struct media_quality *q, uint32_t len_ms); +/* Creates a header for later transcoding of a source file. This header can be + * given to transcode_encode_setup which in some cases will make it faster (MP4) + * + * @out header An evbuffer with the header + * @in profile Transcoding profile + * @in path Path to the source file + * @return Negative if error, otherwise zero + */ +int +transcode_prepare_header(struct evbuffer **header, enum transcode_profile profile, const char *path); + #endif /* !__TRANSCODE_H__ */ From 2efad1466ff292acf812d2833de56a0fdf77de6c Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Fri, 29 Dec 2023 17:44:48 +0100 Subject: [PATCH 05/12] [cache] Add support for storing MP4 headers --- src/cache.c | 740 +++++++++++++++++++++++++++++++----------------- src/cache.h | 11 +- src/httpd.c | 29 ++ src/httpd.h | 11 + src/transcode.c | 2 +- 5 files changed, 525 insertions(+), 268 deletions(-) diff --git a/src/cache.c b/src/cache.c index fe4eedf7..7c003c1a 100644 --- a/src/cache.c +++ b/src/cache.c @@ -44,7 +44,7 @@ #include "commands.h" -#define CACHE_VERSION 3 +#define CACHE_VERSION 4 struct cache_arg @@ -54,6 +54,9 @@ struct cache_arg int is_remote; int msec; + uint32_t id; // file id + const char *header_format; + const char *path; // artwork path char *pathcopy; // copy of artwork path (for async operations) int type; // individual or group artwork @@ -68,6 +71,13 @@ struct cache_arg struct evbuffer *evbuf; }; +struct cachelist +{ + uint32_t id; + uint32_t ts; +}; + + /* --- Globals --- */ // cache thread static pthread_t tid_cache; @@ -76,6 +86,7 @@ static pthread_t tid_cache; struct event_base *evbase_cache; static struct commands_base *cmdbase; static struct event *cache_daap_updateev; +static struct event *cache_xcode_updateev; static int g_initialized; @@ -98,6 +109,96 @@ static int g_suspended; // that will have their reply cached static int g_cfg_threshold; +struct cache_db_def +{ + const char *name; + const char *create_query; + const char *drop_query; +}; + +struct cache_db_def cache_db_def[] = { + { + "xcode_files", + "CREATE TABLE IF NOT EXISTS xcode_files (" + " id INTEGER PRIMARY KEY NOT NULL," + " time_modified INTEGER DEFAULT 0," + " filepath VARCHAR(4096) NOT NULL" + ");", + "DROP TABLE IF EXISTS xcode_files;", + }, + { + "xcode_data", + "CREATE TABLE IF NOT EXISTS xcode_data (" + " id INTEGER PRIMARY KEY NOT NULL," + " timestamp INTEGER DEFAULT 0," + " file_id INTEGER DEFAULT 0," + " format VARCHAR(255) NOT NULL," + " header BLOB" + ");", + "DROP TABLE IF EXISTS xcode_data;", + }, + { + "replies", + "CREATE TABLE IF NOT EXISTS replies (" + " id INTEGER PRIMARY KEY NOT NULL," + " query VARCHAR(4096) NOT NULL," + " reply BLOB" + ");", + "DROP TABLE IF EXISTS replies;", + }, + { + "queries", + "CREATE TABLE IF NOT EXISTS queries (" + " id INTEGER PRIMARY KEY NOT NULL," + " query VARCHAR(4096) UNIQUE NOT NULL," + " user_agent VARCHAR(1024)," + " is_remote INTEGER DEFAULT 0," + " msec INTEGER DEFAULT 0," + " timestamp INTEGER DEFAULT 0" + ");", + "DROP TABLE IF EXISTS queries;", + }, + { + "idx_query", + "CREATE INDEX IF NOT EXISTS idx_query ON replies (query);", + "DROP INDEX IF EXISTS idx_query;", + }, + { + "artwork", + "CREATE TABLE IF NOT EXISTS artwork (" + " id INTEGER PRIMARY KEY NOT NULL," + " type INTEGER NOT NULL DEFAULT 0," + " persistentid INTEGER NOT NULL," + " max_w INTEGER NOT NULL," + " max_h INTEGER NOT NULL," + " format INTEGER NOT NULL," + " filepath VARCHAR(4096) NOT NULL," + " db_timestamp INTEGER DEFAULT 0," + " data BLOB" + ");", + "DROP TABLE IF EXISTS artwork;", + }, + { + "idx_persistentidwh", + "CREATE INDEX IF NOT EXISTS idx_persistentidwh ON artwork(type, persistentid, max_w, max_h);", + "DROP INDEX IF EXISTS idx_persistentidwh;", + }, + { + "idx_pathtime", + "CREATE INDEX IF NOT EXISTS idx_pathtime ON artwork(filepath, db_timestamp);", + "DROP INDEX IF EXISTS idx_pathtime;", + }, + { + "admin_cache", + "CREATE TABLE IF NOT EXISTS admin_cache(" + " key VARCHAR(32) PRIMARY KEY NOT NULL," + " value VARCHAR(32) NOT NULL" + ");", + "DROP TABLE IF EXISTS admin_cache;", + }, +}; + + /* --------------------------------- HELPERS ------------------------------- */ /* The purpose of this function is to remove transient tags from a request @@ -124,128 +225,27 @@ remove_tag(char *in, const char *tag) /* --------------------------------- MAIN --------------------------------- */ /* Thread: cache */ + static int cache_create_tables(void) { -#define T_REPLIES \ - "CREATE TABLE IF NOT EXISTS replies (" \ - " id INTEGER PRIMARY KEY NOT NULL," \ - " query VARCHAR(4096) NOT NULL," \ - " reply BLOB" \ - ");" -#define T_QUERIES \ - "CREATE TABLE IF NOT EXISTS queries (" \ - " id INTEGER PRIMARY KEY NOT NULL," \ - " query VARCHAR(4096) UNIQUE NOT NULL," \ - " user_agent VARCHAR(1024)," \ - " is_remote INTEGER DEFAULT 0," \ - " msec INTEGER DEFAULT 0," \ - " timestamp INTEGER DEFAULT 0" \ - ");" -#define I_QUERY \ - "CREATE INDEX IF NOT EXISTS idx_query ON replies (query);" -#define T_ARTWORK \ - "CREATE TABLE IF NOT EXISTS artwork (" \ - " id INTEGER PRIMARY KEY NOT NULL,"\ - " type INTEGER NOT NULL DEFAULT 0," \ - " persistentid INTEGER NOT NULL," \ - " max_w INTEGER NOT NULL," \ - " max_h INTEGER NOT NULL," \ - " format INTEGER NOT NULL," \ - " filepath VARCHAR(4096) NOT NULL," \ - " db_timestamp INTEGER DEFAULT 0," \ - " data BLOB" \ - ");" -#define I_ARTWORK_ID \ - "CREATE INDEX IF NOT EXISTS idx_persistentidwh ON artwork(type, persistentid, max_w, max_h);" -#define I_ARTWORK_PATH \ - "CREATE INDEX IF NOT EXISTS idx_pathtime ON artwork(filepath, db_timestamp);" -#define T_ADMIN_CACHE \ - "CREATE TABLE IF NOT EXISTS admin_cache(" \ - " key VARCHAR(32) PRIMARY KEY NOT NULL," \ - " value VARCHAR(32) NOT NULL" \ - ");" -#define Q_CACHE_VERSION \ - "INSERT INTO admin_cache (key, value) VALUES ('cache_version', '%d');" - +#define Q_CACHE_VERSION "INSERT INTO admin_cache (key, value) VALUES ('cache_version', '%d');" char *query; char *errmsg; int ret; + int i; - - // Create reply cache table - ret = sqlite3_exec(g_db_hdl, T_REPLIES, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) + for (i = 0; i < ARRAY_SIZE(cache_db_def); i++) { - DPRINTF(E_FATAL, L_CACHE, "Error creating cache table 'replies': %s\n", errmsg); + ret = sqlite3_exec(g_db_hdl, cache_db_def[i].create_query, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_FATAL, L_CACHE, "Error creating cache db entity '%s': %s\n", cache_db_def[i].name, errmsg); - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Create query table (the queries for which we will generate and cache replies) - ret = sqlite3_exec(g_db_hdl, T_QUERIES, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating cache table 'queries': %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Create index - ret = sqlite3_exec(g_db_hdl, I_QUERY, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating index on replies(query): %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Create artwork table - ret = sqlite3_exec(g_db_hdl, T_ARTWORK, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating cache table 'artwork': %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Create index - ret = sqlite3_exec(g_db_hdl, I_ARTWORK_ID, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating index on artwork(type, persistentid, max_w, max_h): %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - ret = sqlite3_exec(g_db_hdl, I_ARTWORK_PATH, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating index on artwork(filepath, db_timestamp): %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Create admin cache table - ret = sqlite3_exec(g_db_hdl, T_ADMIN_CACHE, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error creating cache table 'admin_cache': %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; + sqlite3_free(errmsg); + sqlite3_close(g_db_hdl); + return -1; + } } query = sqlite3_mprintf(Q_CACHE_VERSION, CACHE_VERSION); @@ -263,108 +263,30 @@ cache_create_tables(void) DPRINTF(E_DBG, L_CACHE, "Cache tables created\n"); return 0; -#undef T_REPLIES -#undef T_QUERIES -#undef I_QUERY -#undef T_ARTWORK -#undef I_ARTWORK_ID -#undef I_ARTWORK_PATH -#undef T_ADMIN_CACHE #undef Q_CACHE_VERSION } static int cache_drop_tables(void) { -#define D_REPLIES "DROP TABLE IF EXISTS replies;" -#define D_QUERIES "DROP TABLE IF EXISTS queries;" -#define D_QUERY "DROP INDEX IF EXISTS idx_query;" -#define D_ARTWORK "DROP TABLE IF EXISTS artwork;" -#define D_ARTWORK_ID "DROP INDEX IF EXISTS idx_persistentidwh;" -#define D_ARTWORK_PATH "DROP INDEX IF EXISTS idx_pathtime;" -#define D_ADMIN_CACHE "DROP TABLE IF EXISTS admin_cache;" #define Q_VACUUM "VACUUM;" - char *errmsg; int ret; + int i; - - // Drop reply cache table - ret = sqlite3_exec(g_db_hdl, D_REPLIES, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) + for (i = 0; i < ARRAY_SIZE(cache_db_def); i++) { - DPRINTF(E_FATAL, L_CACHE, "Error dropping reply cache table: %s\n", errmsg); + ret = sqlite3_exec(g_db_hdl, cache_db_def[i].drop_query, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_FATAL, L_CACHE, "Error dropping cache db entity '%s': %s\n", cache_db_def[i].name, errmsg); - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; + sqlite3_free(errmsg); + sqlite3_close(g_db_hdl); + return -1; + } } - // Drop query table - ret = sqlite3_exec(g_db_hdl, D_QUERIES, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping query table: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Drop index - ret = sqlite3_exec(g_db_hdl, D_QUERY, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping query index: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Drop artwork table - ret = sqlite3_exec(g_db_hdl, D_ARTWORK, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping artwork table: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Drop index - ret = sqlite3_exec(g_db_hdl, D_ARTWORK_ID, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping artwork id index: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - ret = sqlite3_exec(g_db_hdl, D_ARTWORK_PATH, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping artwork path index: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Drop admin cache table - ret = sqlite3_exec(g_db_hdl, D_ADMIN_CACHE, NULL, NULL, &errmsg); - if (ret != SQLITE_OK) - { - DPRINTF(E_FATAL, L_CACHE, "Error dropping admin cache table: %s\n", errmsg); - - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); - return -1; - } - - // Vacuum ret = sqlite3_exec(g_db_hdl, Q_VACUUM, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { @@ -378,13 +300,6 @@ cache_drop_tables(void) DPRINTF(E_DBG, L_CACHE, "Cache tables dropped\n"); return 0; -#undef D_REPLIES -#undef D_QUERIES -#undef D_QUERY -#undef D_ARTWORK -#undef D_ARTWORK_ID -#undef D_ARTWORK_PATH -#undef D_ADMIN_CACHE #undef Q_VACUUM } @@ -876,15 +791,329 @@ cache_daap_update_cb(int fd, short what, void *arg) DPRINTF(E_LOG, L_CACHE, "DAAP cache updated\n"); } +static enum command_state +xcode_header_get(void *arg, int *retval) +{ +#define Q_TMPL "SELECT header FROM xcode_data WHERE length(header) > 0 AND id = ? AND format = ?;" + struct cache_arg *cmdarg = arg; + sqlite3_stmt *stmt = NULL; + int ret; + + cmdarg->cached = 0; + + ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + if (ret != SQLITE_OK) + goto error; + + sqlite3_bind_int(stmt, 1, cmdarg->id); + sqlite3_bind_text(stmt, 2, cmdarg->header_format, -1, SQLITE_STATIC); + + ret = sqlite3_step(stmt); + if (ret == SQLITE_DONE) + goto end; + else if (ret != SQLITE_ROW) + goto error; + + ret = evbuffer_add(cmdarg->evbuf, sqlite3_column_blob(stmt, 0), sqlite3_column_bytes(stmt, 0)); + if (ret < 0) + goto error; + + cmdarg->cached = 1; + + DPRINTF(E_DBG, L_CACHE, "Cache header hit (%zu bytes)\n", evbuffer_get_length(cmdarg->evbuf)); + + end: + sqlite3_finalize(stmt); + *retval = 0; + return COMMAND_END; + + error: + DPRINTF(E_LOG, L_CACHE, "Database error getting prepared header from cache: %s\n", sqlite3_errmsg(g_db_hdl)); + if (stmt) + sqlite3_finalize(stmt); + *retval = -1; + return COMMAND_END; +#undef Q_TMPL +} + +static int +xcode_add_entry(uint32_t id, uint32_t ts, const char *path) +{ +#define Q_TMPL "INSERT OR REPLACE INTO xcode_files (id, time_modified, filepath) VALUES (%d, %d, '%q');" + char *query; + char *errmsg; + int ret; + + DPRINTF(E_LOG, L_CACHE, "Adding xcode file id %d, path '%s'\n", id, path); + + query = sqlite3_mprintf(Q_TMPL, id, ts, path); + + ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + sqlite3_free(query); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_CACHE, "Error adding row to cache: %s\n", errmsg); + sqlite3_free(errmsg); + return -1; + } + + return 0; +#undef Q_TMPL +} + +static int +xcode_del_entry(uint32_t id) +{ +#define Q_TMPL_FILES "DELETE FROM xcode_files WHERE id = %d;" +#define Q_TMPL_DATA "DELETE FROM xcode_data WHERE file_id = %d;" + char query[256]; + char *errmsg; + int ret; + + DPRINTF(E_LOG, L_CACHE, "Deleting xcode file id %d\n", id); + + sqlite3_snprintf(sizeof(query), query, Q_TMPL_FILES, (int)id); + ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_CACHE, "Error deleting row from xcode_files: %s\n", errmsg); + sqlite3_free(errmsg); + return -1; + } + + sqlite3_snprintf(sizeof(query), query, Q_TMPL_DATA, (int)id); + ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_CACHE, "Error deleting rows from xcode_data: %s\n", errmsg); + sqlite3_free(errmsg); + return -1; + } + + return 0; +#undef Q_TMPL_DATA +#undef Q_TMPL_FILES +} + +/* In the xcode table we keep a prepared header for files that could be subject + * to transcoding. Whenever the library changes, this callback runs, and the + * list of files in the xcode table is synced with the main files table. + * + * In practice we compare two tables, both sorted by id: + * + * From files: From the cache + * | id | time_modified | | id | time_modified | data | + * + * We do it one item at the time from files, and then going through cache table + * rows until: table end OR id is larger OR id is equal and time equal or newer + */ +static int +xcode_sync_with_files(void) +{ + sqlite3_stmt *stmt; + struct cachelist *cachelist = NULL; + size_t cachelist_size = 0; + size_t cachelist_len = 0; + struct query_params qp = { .type = Q_ITEMS, .filter = "f.data_kind = 0", .order = "f.id" }; + struct db_media_file_info dbmfi; + uint32_t id; + uint32_t ts; + int i; + int ret; + + DPRINTF(E_LOG, L_CACHE, "SYNC START\n"); + + // Both lists must be sorted by id, otherwise the compare below won't work + ret = sqlite3_prepare_v2(g_db_hdl, "SELECT id, time_modified FROM xcode_files ORDER BY id;", -1, &stmt, 0); + if (ret != SQLITE_OK) + goto error; + + while (sqlite3_step(stmt) == SQLITE_ROW) + { + if (cachelist_len + 1 > cachelist_size) + { + cachelist_size += 1024; + CHECK_NULL(L_CACHE, cachelist = realloc(cachelist, cachelist_size * sizeof(struct cachelist))); + } + cachelist[cachelist_len].id = sqlite3_column_int(stmt, 0); + cachelist[cachelist_len].ts = sqlite3_column_int(stmt, 1); + cachelist_len++; + } + sqlite3_finalize(stmt); + + ret = db_query_start(&qp); + if (ret < 0) + goto error; + + // Loop while either list has remaining items + i = 0; + while (1) + { + ret = db_query_fetch_file(&dbmfi, &qp); + if (ret != 0) // At end of files table (or error occured) + { + for (; i < cachelist_len; i++) + xcode_del_entry(cachelist[i].id); + + break; + } + + safe_atou32(dbmfi.id, &id); + safe_atou32(dbmfi.time_modified, &ts); + + if (i == cachelist_len || cachelist[i].id > id) // At end of cache table or new file + { + xcode_add_entry(id, ts, dbmfi.path); + } + else if (cachelist[i].id < id) // Removed file + { + xcode_del_entry(cachelist[i].id); + i++; + } + else if (cachelist[i].id == id && cachelist[i].ts < ts) // Modified file + { + xcode_del_entry(cachelist[i].id); + xcode_add_entry(id, ts, dbmfi.path); + i++; + } + else // Found in both tables and timestamp in cache table is adequate + { + i++; + } + } + db_query_end(&qp); + + free(cachelist); + return 0; + + error: + DPRINTF(E_LOG, L_CACHE, "Database error while processing xcode_files table\n"); + free(cachelist); + return -1; +} + +static int +xcode_prepare_header(const char *format, int id, const char *path) +{ +#define Q_TMPL "INSERT INTO xcode_data (timestamp, file_id, format, header) VALUES (?, ?, ?, ?);" + struct evbuffer *header = NULL; + sqlite3_stmt *stmt = NULL; + unsigned char *data = NULL; + size_t datalen = 0; + int ret; + + DPRINTF(E_DBG, L_CACHE, "Preparing %s header for '%s' (file id %d)\n", format, path, id); + +#if 1 + ret = httpd_prepare_header(&header, format, path); // Proceed even if error, we also cache that + if (ret == 0) + { + datalen = evbuffer_get_length(header); + data = evbuffer_pullup(header, -1); + } +#elif + data = (unsigned char*)"dummy"; + datalen = 6; +#endif + + ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_CACHE, "Error preparing xcode_data for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + goto error; + } + + sqlite3_bind_int(stmt, 1, (uint64_t)time(NULL)); + sqlite3_bind_int(stmt, 2, id); + sqlite3_bind_text(stmt, 3, format, -1, SQLITE_STATIC); + sqlite3_bind_blob(stmt, 4, data, datalen, SQLITE_STATIC); + + ret = sqlite3_step(stmt); + if (ret != SQLITE_DONE) + { + DPRINTF(E_LOG, L_CACHE, "Error stepping xcode_data for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + goto error; + } + + sqlite3_finalize(stmt); + if (header) + evbuffer_free(header); + return 0; + + error: + if (stmt) + sqlite3_finalize(stmt); + if (header) + evbuffer_free(header); + return -1; +#undef Q_TMPL +} + +static int +xcode_prepare_headers(const char *format) +{ +#define Q_TMPL "SELECT xf.id, xf.filepath, xd.id FROM xcode_files xf LEFT JOIN xcode_data xd ON xf.id = xd.file_id AND xd.format = '%q';" + sqlite3_stmt *stmt; + char *query; + const char *file_path; + int file_id; + int data_id; + int ret; + + query = sqlite3_mprintf(Q_TMPL, format); + + ret = sqlite3_prepare_v2(g_db_hdl, query, -1, &stmt, 0); + if (ret != SQLITE_OK) + goto error; + + while (sqlite3_step(stmt) == SQLITE_ROW) + { + data_id = sqlite3_column_int(stmt, 2); + if (data_id > 0) + continue; // Already have a prepared header + + file_id = sqlite3_column_int(stmt, 0); + file_path = (const char *)sqlite3_column_text(stmt, 1); + + xcode_prepare_header(format, file_id, file_path); + + } + sqlite3_finalize(stmt); + sqlite3_free(query); + return 0; + + error: + DPRINTF(E_LOG, L_CACHE, "Error occured while preparing headers\n"); + sqlite3_free(query); + return -1; +#undef Q_TMPL +} + +static void +cache_xcode_update_cb(int fd, short what, void *arg) +{ + if (xcode_sync_with_files() < 0) + return; + + xcode_prepare_headers("mp4"); +} + /* Sets off an update by activating the event. The delay is because we are low * priority compared to other listeners of database updates. */ static enum command_state -cache_daap_update(void *arg, int *retval) +cache_database_update(void *arg, int *retval) { - struct timeval delay = { 10, 0 }; + struct timeval delay_daap = { 10, 0 }; + struct timeval delay_xcode = { 5, 0 }; +// const char *prefer_format = cfg_getstr(cfg_getsec(cfg, "library"), "prefer_format"); - *retval = event_add(cache_daap_updateev, &delay); + event_add(cache_daap_updateev, &delay_daap); + +// if (prefer_format && strcmp(prefer_format, "alac")) // TODO Ugly + event_add(cache_xcode_updateev, &delay_xcode); + + *retval = 0; return COMMAND_END; } @@ -892,7 +1121,7 @@ cache_daap_update(void *arg, int *retval) static void cache_daap_listener_cb(short event_mask) { - commands_exec_async(cmdbase, cache_daap_update, NULL); + commands_exec_async(cmdbase, cache_database_update, NULL); } @@ -1290,7 +1519,7 @@ cache(void *arg) } /* The thread needs a connection with the main db, so it can generate DAAP - * replies through httpd_daap.c + * replies through httpd_daap.c and read changes from the files table */ ret = db_perthread_init(); if (ret < 0) @@ -1319,7 +1548,7 @@ cache(void *arg) } -/* ---------------------------- DAAP cache API --------------------------- */ +/* ----------------------------- DAAP cache API ---------------------------- */ /* The DAAP cache will cache raw daap replies for queries added with * cache_daap_add(). Only some query types are supported. @@ -1384,7 +1613,30 @@ cache_daap_threshold(void) } -/* --------------------------- Artwork cache API -------------------------- */ +/* --------------------------- Transcode cache API ------------------------- */ + +int +cache_xcode_header_get(struct evbuffer *evbuf, int *cached, uint32_t id, const char *format) +{ + struct cache_arg cmdarg; + int ret; + + if (!g_initialized) + return -1; + + cmdarg.evbuf = evbuf; + cmdarg.id = id; + cmdarg.header_format = format; + + ret = commands_exec_sync(cmdbase, xcode_header_get, NULL, &cmdarg); + + *cached = cmdarg.cached; + + return ret; +} + + +/* ---------------------------- Artwork cache API -------------------------- */ /* * Updates cached timestamps to current time for all cache entries for the given path, if the file was not modfied @@ -1581,15 +1833,11 @@ cache_artwork_read(struct evbuffer *evbuf, const char *path, int *format) } -/* -------------------------- Cache general API --------------------------- */ +/* --------------------------- Cache general API ---------------------------- */ int cache_init(void) { - int ret; - - g_initialized = 0; - g_db_path = cfg_getstr(cfg_getsec(cfg, "general"), "cache_path"); if (!g_db_path || (strlen(g_db_path) == 0)) { @@ -1604,53 +1852,17 @@ cache_init(void) return 0; } - evbase_cache = event_base_new(); - if (!evbase_cache) - { - DPRINTF(E_LOG, L_CACHE, "Could not create an event base\n"); - goto evbase_fail; - } - - cache_daap_updateev = evtimer_new(evbase_cache, cache_daap_update_cb, NULL); - if (!cache_daap_updateev) - { - DPRINTF(E_LOG, L_CACHE, "Could not create cache event\n"); - goto evnew_fail; - } - - cmdbase = commands_base_new(evbase_cache, NULL); - - ret = listener_add(cache_daap_listener_cb, LISTENER_DATABASE); - if (ret < 0) - { - DPRINTF(E_LOG, L_CACHE, "Could not create listener event\n"); - goto listener_fail; - } + CHECK_NULL(L_CACHE, evbase_cache = event_base_new()); + CHECK_NULL(L_CACHE, cache_daap_updateev = evtimer_new(evbase_cache, cache_daap_update_cb, NULL)); + CHECK_NULL(L_CACHE, cache_xcode_updateev = evtimer_new(evbase_cache, cache_xcode_update_cb, NULL)); + CHECK_NULL(L_CACHE, cmdbase = commands_base_new(evbase_cache, NULL)); + CHECK_ERR(L_CACHE, listener_add(cache_daap_listener_cb, LISTENER_DATABASE)); + CHECK_ERR(L_CACHE, pthread_create(&tid_cache, NULL, cache, NULL)); + thread_setname(tid_cache, "cache"); DPRINTF(E_INFO, L_CACHE, "cache thread init\n"); - ret = pthread_create(&tid_cache, NULL, cache, NULL); - if (ret < 0) - { - DPRINTF(E_LOG, L_CACHE, "Could not spawn cache thread: %s\n", strerror(errno)); - - goto thread_fail; - } - - thread_setname(tid_cache, "cache"); - return 0; - - thread_fail: - listener_remove(cache_daap_listener_cb); - listener_fail: - commands_base_free(cmdbase); - evnew_fail: - event_base_free(evbase_cache); - evbase_cache = NULL; - - evbase_fail: - return -1; } void diff --git a/src/cache.h b/src/cache.h index acf8bfbb..036e09d7 100644 --- a/src/cache.h +++ b/src/cache.h @@ -4,7 +4,7 @@ #include -/* ---------------------------- DAAP cache API --------------------------- */ +/* ----------------------------- DAAP cache API ---------------------------- */ void cache_daap_suspend(void); @@ -22,7 +22,12 @@ int cache_daap_threshold(void); -/* ---------------------------- Artwork cache API --------------------------- */ +/* --------------------------- Transcode cache API ------------------------- */ + +int +cache_xcode_header_get(struct evbuffer *evbuf, int *cached, uint32_t id, const char *format); + +/* ---------------------------- Artwork cache API -------------------------- */ #define CACHE_ARTWORK_GROUP 0 #define CACHE_ARTWORK_INDIVIDUAL 1 @@ -48,7 +53,7 @@ cache_artwork_stash(struct evbuffer *evbuf, const char *path, int format); int cache_artwork_read(struct evbuffer *evbuf, const char *path, int *format); -/* ---------------------------- Cache API --------------------------- */ +/* ------------------------------- Cache API ------------------------------- */ int cache_init(void); diff --git a/src/httpd.c b/src/httpd.c index d2f3fb96..07527ed7 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -48,6 +48,7 @@ #include "httpd.h" #include "httpd_internal.h" #include "transcode.h" +#include "cache.h" #ifdef LASTFM # include "lastfm.h" #endif @@ -676,7 +677,10 @@ stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile struct transcode_decode_setup_args decode_args = { 0 }; struct transcode_encode_setup_args encode_args = { 0 }; struct media_quality quality = { 0 }; + struct evbuffer *prepared_header = NULL; struct stream_ctx *st; + int cached; + int ret; // We use source sample rate etc, but for MP3 we must set a bit rate quality.bit_rate = 1000 * cfg_getint(cfg_getsec(cfg, "streaming"), "bit_rate"); @@ -687,12 +691,25 @@ stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile goto error; } + if (profile == XCODE_MP4_ALAC) + { + CHECK_NULL(L_HTTPD, prepared_header = evbuffer_new()); + + ret = cache_xcode_header_get(prepared_header, &cached, mfi->id, "mp4"); + if (ret < 0 || !cached) // Error or not found + { + evbuffer_free(prepared_header); + prepared_header = NULL; + } + } + decode_args.profile = profile; decode_args.is_http = (mfi->data_kind == DATA_KIND_HTTP); decode_args.path = mfi->path; decode_args.len_ms = mfi->song_length; encode_args.profile = profile; encode_args.quality = &quality; + encode_args.prepared_header = prepared_header; st->xcode = transcode_setup(decode_args, encode_args); if (!st->xcode) @@ -718,9 +735,13 @@ stream_new_transcode(struct media_file_info *mfi, enum transcode_profile profile st->start_offset = offset; + if (prepared_header) + evbuffer_free(prepared_header); return st; error: + if (prepared_header) + evbuffer_free(prepared_header); stream_free(st); return NULL; } @@ -1199,6 +1220,14 @@ httpd_gzip_deflate(struct evbuffer *in) return NULL; } +int +httpd_prepare_header(struct evbuffer **header, const char *format, const char *path) +{ + if (strcmp(format, "mp4") == 0) + return transcode_prepare_header(header, XCODE_MP4_ALAC, path); + else + return -1; +} // The httpd_send functions below can be called from a worker thread (with // hreq->is_async) or directly from the httpd thread. In the former case, they diff --git a/src/httpd.h b/src/httpd.h index 8926f707..0fed50c6 100644 --- a/src/httpd.h +++ b/src/httpd.h @@ -13,6 +13,17 @@ struct evbuffer * httpd_gzip_deflate(struct evbuffer *in); +/* + * Passthrough to transcode, which will create a transcoded file header for path + * + * @out header Newly created evbuffer with the header + * @in format Which format caller wants a header for + * @in path Path to the file + * @return 0 if ok, otherwise -1 + */ +int +httpd_prepare_header(struct evbuffer **header, const char *format, const char *path); + int httpd_init(const char *webroot); diff --git a/src/transcode.c b/src/transcode.c index abcfb0d8..37c1d595 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -2625,7 +2625,7 @@ transcode_metadata_strings_set(struct transcode_metadata_string *s, enum transco } int -transcode_create_header(struct evbuffer **header, enum transcode_profile profile, const char *path) +transcode_prepare_header(struct evbuffer **header, enum transcode_profile profile, const char *path) { int ret; From c079df5da7996166120081f36d180a12b77185d7 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Sat, 6 Jan 2024 00:27:34 +0100 Subject: [PATCH 06/12] [cache/config] Refactor cache so daap/artwork/xcode is cached in separate db's Also change config so that the user can just configure a data directory instead of complete path to each database. --- owntone.conf.in | 7 +- src/cache.c | 729 ++++++++++++++++++++++++++--------------------- src/cache.h | 2 +- src/conffile.c | 39 ++- src/db.c | 20 +- src/httpd_daap.c | 2 +- 6 files changed, 461 insertions(+), 338 deletions(-) diff --git a/owntone.conf.in b/owntone.conf.in index 30d67b19..21966ea6 100644 --- a/owntone.conf.in +++ b/owntone.conf.in @@ -52,8 +52,8 @@ general { # IP addresses. # bind_address = "::" - # Location of cache database -# cache_path = "@localstatedir@/cache/@PACKAGE@/cache.db" + # Directory where the server keeps cached data +# cache_dir = "@localstatedir@/cache/@PACKAGE@" # DAAP requests that take longer than this threshold (in msec) get their # replies cached for next time. Set to 0 to disable caching. @@ -203,7 +203,8 @@ library { # Formats that should always be transcoded # force_decode = { "format", "format" } # Prefer transcode to wav (default), alac or mpeg (mp3 with the bit rate - # configured below in the streaming section) + # configured below in the streaming section). Note that alac requires + # precomputing and caching mp4 headers, which takes both cpu and disk. # prefer_format = "format" # Set ffmpeg filters (similar to 'ffmpeg -af xxx') that you want the diff --git a/src/cache.c b/src/cache.c index 7c003c1a..f61a7001 100644 --- a/src/cache.c +++ b/src/cache.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -43,12 +44,10 @@ #include "listener.h" #include "commands.h" - -#define CACHE_VERSION 4 - - struct cache_arg { + sqlite3 *hdl; // which cache database + char *query; // daap query char *ua; // user agent int is_remote; @@ -77,38 +76,6 @@ struct cachelist uint32_t ts; }; - -/* --- Globals --- */ -// cache thread -static pthread_t tid_cache; - -// Event base, pipes and events -struct event_base *evbase_cache; -static struct commands_base *cmdbase; -static struct event *cache_daap_updateev; -static struct event *cache_xcode_updateev; - -static int g_initialized; - -// Global cache database handle -static sqlite3 *g_db_hdl; -static char *g_db_path; - -// Global artwork stash -struct stash -{ - char *path; - int format; - size_t size; - uint8_t *data; -} g_stash; - -static int g_suspended; - -// The user may configure a threshold (in msec), and queries slower than -// that will have their reply cached -static int g_cfg_threshold; - struct cache_db_def { const char *name; @@ -116,27 +83,46 @@ struct cache_db_def const char *drop_query; }; -struct cache_db_def cache_db_def[] = { - { - "xcode_files", - "CREATE TABLE IF NOT EXISTS xcode_files (" - " id INTEGER PRIMARY KEY NOT NULL," - " time_modified INTEGER DEFAULT 0," - " filepath VARCHAR(4096) NOT NULL" - ");", - "DROP TABLE IF EXISTS xcode_files;", - }, - { - "xcode_data", - "CREATE TABLE IF NOT EXISTS xcode_data (" - " id INTEGER PRIMARY KEY NOT NULL," - " timestamp INTEGER DEFAULT 0," - " file_id INTEGER DEFAULT 0," - " format VARCHAR(255) NOT NULL," - " header BLOB" - ");", - "DROP TABLE IF EXISTS xcode_data;", - }, +struct cache_artwork_stash +{ + char *path; + int format; + size_t size; + uint8_t *data; +}; + +/* --------------------------------- GLOBALS -------------------------------- */ + +// cache thread +static pthread_t tid_cache; + +// Event base, pipes and events +static struct event_base *evbase_cache; +static struct commands_base *cmdbase; + +// State +static bool cache_is_initialized; +static bool cache_is_suspended; + +#define DB_DEF_ADMIN \ + { \ + "admin", \ + "CREATE TABLE IF NOT EXISTS admin(" \ + " key VARCHAR(32) PRIMARY KEY NOT NULL," \ + " value VARCHAR(32) NOT NULL" \ + ");", \ + "DROP TABLE IF EXISTS admin;", \ + } + +// DAAP cache +#define CACHE_DAAP_VERSION 5 +static sqlite3 *cache_daap_hdl; +static struct event *cache_daap_updateev; +// The user may configure a threshold (in msec), and queries slower than +// that will have their reply cached +static int cache_daap_threshold; +static struct cache_db_def cache_daap_db_def[] = { + DB_DEF_ADMIN, { "replies", "CREATE TABLE IF NOT EXISTS replies (" @@ -163,6 +149,14 @@ struct cache_db_def cache_db_def[] = { "CREATE INDEX IF NOT EXISTS idx_query ON replies (query);", "DROP INDEX IF EXISTS idx_query;", }, +}; + +// Artwork cache +#define CACHE_ARTWORK_VERSION 5 +static sqlite3 *cache_artwork_hdl; +static struct cache_artwork_stash cache_stash; +static struct cache_db_def cache_artwork_db_def[] = { + DB_DEF_ADMIN, { "artwork", "CREATE TABLE IF NOT EXISTS artwork (" @@ -188,18 +182,40 @@ struct cache_db_def cache_db_def[] = { "CREATE INDEX IF NOT EXISTS idx_pathtime ON artwork(filepath, db_timestamp);", "DROP INDEX IF EXISTS idx_pathtime;", }, +}; + +// Transcoding cache +#define CACHE_XCODE_VERSION 1 +static sqlite3 *cache_xcode_hdl; +static struct event *cache_xcode_updateev; +static struct event *cache_xcode_prepareev; +static int cache_xcode_last_file; +static struct cache_db_def cache_xcode_db_def[] = { + DB_DEF_ADMIN, { - "admin_cache", - "CREATE TABLE IF NOT EXISTS admin_cache(" - " key VARCHAR(32) PRIMARY KEY NOT NULL," - " value VARCHAR(32) NOT NULL" + "files", + "CREATE TABLE IF NOT EXISTS files (" + " id INTEGER PRIMARY KEY NOT NULL," + " time_modified INTEGER DEFAULT 0," + " filepath VARCHAR(4096) NOT NULL" ");", - "DROP TABLE IF EXISTS admin_cache;", + "DROP TABLE IF EXISTS files;", + }, + { + "data", + "CREATE TABLE IF NOT EXISTS data (" + " id INTEGER PRIMARY KEY NOT NULL," + " timestamp INTEGER DEFAULT 0," + " file_id INTEGER DEFAULT 0," + " format VARCHAR(255) NOT NULL," + " header BLOB" + ");", + "DROP TABLE IF EXISTS data;", }, }; -/* --------------------------------- HELPERS ------------------------------- */ +/* --------------------------------- HELPERS -------------------------------- */ /* The purpose of this function is to remove transient tags from a request * url (query), eg remove session-id=xxx @@ -222,138 +238,106 @@ remove_tag(char *in, const char *tag) } -/* --------------------------------- MAIN --------------------------------- */ -/* Thread: cache */ - +/* ---------------------------------- MAIN ---------------------------------- */ +/* Thread: cache */ static int -cache_create_tables(void) +cache_tables_create(sqlite3 *hdl, int version, struct cache_db_def *db_def, int db_def_size) { -#define Q_CACHE_VERSION "INSERT INTO admin_cache (key, value) VALUES ('cache_version', '%d');" +#define Q_CACHE_VERSION "INSERT INTO admin (key, value) VALUES ('cache_version', '%d');" char *query; char *errmsg; int ret; int i; - for (i = 0; i < ARRAY_SIZE(cache_db_def); i++) + for (i = 0; i < db_def_size; i++) { - ret = sqlite3_exec(g_db_hdl, cache_db_def[i].create_query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, db_def[i].create_query, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { - DPRINTF(E_FATAL, L_CACHE, "Error creating cache db entity '%s': %s\n", cache_db_def[i].name, errmsg); - + DPRINTF(E_FATAL, L_CACHE, "Error creating cache db entity '%s': %s\n", db_def[i].name, errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } - query = sqlite3_mprintf(Q_CACHE_VERSION, CACHE_VERSION); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + query = sqlite3_mprintf(Q_CACHE_VERSION, version); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { DPRINTF(E_FATAL, L_CACHE, "Error inserting cache version: %s\n", errmsg); - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } - DPRINTF(E_DBG, L_CACHE, "Cache tables created\n"); - return 0; #undef Q_CACHE_VERSION } static int -cache_drop_tables(void) +cache_tables_drop(sqlite3 *hdl, struct cache_db_def *db_def, int db_def_size) { #define Q_VACUUM "VACUUM;" char *errmsg; int ret; int i; - for (i = 0; i < ARRAY_SIZE(cache_db_def); i++) + for (i = 0; i < db_def_size; i++) { - ret = sqlite3_exec(g_db_hdl, cache_db_def[i].drop_query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, db_def[i].drop_query, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { - DPRINTF(E_FATAL, L_CACHE, "Error dropping cache db entity '%s': %s\n", cache_db_def[i].name, errmsg); - + DPRINTF(E_FATAL, L_CACHE, "Error dropping cache db entity '%s': %s\n", db_def[i].name, errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } - ret = sqlite3_exec(g_db_hdl, Q_VACUUM, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, Q_VACUUM, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error vacuuming cache database: %s\n", errmsg); - sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } - DPRINTF(E_DBG, L_CACHE, "Cache tables dropped\n"); - return 0; #undef Q_VACUUM } -/* - * Compares the CACHE_VERSION against the version stored in the cache admin table. - * Drops the tables and indexes if the versions are different. - * - * @return 0 if versions are equal, 1 if versions are different or the admin table does not exist, -1 if an error occurred - */ static int -cache_check_version(void) +cache_version_check(int *have_version, sqlite3 *hdl, int want_version) { -#define Q_VER "SELECT value FROM admin_cache WHERE key = 'cache_version';" +#define Q_VER "SELECT value FROM admin WHERE key = 'cache_version';" sqlite3_stmt *stmt; - int cur_ver; int ret; - DPRINTF(E_DBG, L_CACHE, "Running query '%s'\n", Q_VER); + *have_version = 0; - ret = sqlite3_prepare_v2(g_db_hdl, Q_VER, strlen(Q_VER) + 1, &stmt, NULL); + ret = sqlite3_prepare_v2(hdl, Q_VER, -1, &stmt, NULL); if (ret != SQLITE_OK) { - DPRINTF(E_WARN, L_CACHE, "Could not prepare statement: %s\n", sqlite3_errmsg(g_db_hdl)); - return 1; + return 0; // Virgin database, admin table doesn't exists } ret = sqlite3_step(stmt); if (ret != SQLITE_ROW) { - DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(hdl)); sqlite3_finalize(stmt); return -1; } - cur_ver = sqlite3_column_int(stmt, 0); + *have_version = sqlite3_column_int(stmt, 0); sqlite3_finalize(stmt); - if (cur_ver != CACHE_VERSION) - { - DPRINTF(E_LOG, L_CACHE, "Database schema outdated, deleting cache v%d -> v%d\n", cur_ver, CACHE_VERSION); - ret = cache_drop_tables(); - if (ret < 0) - { - DPRINTF(E_LOG, L_CACHE, "Error deleting database tables\n"); - return -1; - } - return 1; - } return 0; #undef Q_VER } static int -cache_create(void) +cache_pragma_set(sqlite3 *hdl) { #define Q_PRAGMA_CACHE_SIZE "PRAGMA cache_size=%d;" #define Q_PRAGMA_JOURNAL_MODE "PRAGMA journal_mode=%s;" @@ -367,50 +351,18 @@ cache_create(void) int mmap_size; char *query; - // Open db - ret = sqlite3_open(g_db_path, &g_db_hdl); - if (ret != SQLITE_OK) - { - DPRINTF(E_LOG, L_CACHE, "Could not open '%s': %s\n", g_db_path, sqlite3_errmsg(g_db_hdl)); - - sqlite3_close(g_db_hdl); - return -1; - } - - // Check cache version - ret = cache_check_version(); - if (ret < 0) - { - DPRINTF(E_LOG, L_CACHE, "Could not check cache database version\n"); - - sqlite3_close(g_db_hdl); - return -1; - } - else if (ret > 0) - { - ret = cache_create_tables(); - if (ret < 0) - { - DPRINTF(E_LOG, L_CACHE, "Could not create cache database tables\n"); - - sqlite3_close(g_db_hdl); - return -1; - } - } - // Set page cache size in number of pages cache_size = cfg_getint(cfg_getsec(cfg, "sqlite"), "pragma_cache_size_cache"); if (cache_size > -1) { query = sqlite3_mprintf(Q_PRAGMA_CACHE_SIZE, cache_size); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error setting pragma_cache_size_cache: %s\n", errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } @@ -420,14 +372,13 @@ cache_create(void) if (journal_mode) { query = sqlite3_mprintf(Q_PRAGMA_JOURNAL_MODE, journal_mode); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error setting pragma_journal_mode: %s\n", errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } @@ -437,14 +388,13 @@ cache_create(void) if (synchronous > -1) { query = sqlite3_mprintf(Q_PRAGMA_SYNCHRONOUS, synchronous); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error setting pragma_synchronous: %s\n", errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } @@ -454,19 +404,16 @@ cache_create(void) if (synchronous > -1) { query = sqlite3_mprintf(Q_PRAGMA_MMAP_SIZE, mmap_size); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error setting pragma_mmap_size: %s\n", errmsg); sqlite3_free(errmsg); - sqlite3_close(g_db_hdl); return -1; } } - DPRINTF(E_DBG, L_CACHE, "Cache created\n"); - return 0; #undef Q_PRAGMA_CACHE_SIZE #undef Q_PRAGMA_JOURNAL_MODE @@ -475,25 +422,140 @@ cache_create(void) } static void -cache_close(void) +cache_close_one(sqlite3 **hdl) { sqlite3_stmt *stmt; - if (!g_db_hdl) + if (!*hdl) return; /* Tear down anything that's in flight */ - while ((stmt = sqlite3_next_stmt(g_db_hdl, 0))) + while ((stmt = sqlite3_next_stmt(*hdl, 0))) sqlite3_finalize(stmt); - sqlite3_close(g_db_hdl); + sqlite3_close(*hdl); + *hdl = NULL; +} + +static void +cache_close(void) +{ + cache_close_one(&cache_daap_hdl); + cache_close_one(&cache_artwork_hdl); + cache_close_one(&cache_xcode_hdl); DPRINTF(E_DBG, L_CACHE, "Cache closed\n"); } +static int +cache_open_one(sqlite3 **hdl, const char *path, const char *name, int want_version, struct cache_db_def *db_def, int db_def_size) +{ + sqlite3 *h; + int have_version; + int ret; + + ret = sqlite3_open(path, &h); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_CACHE, "Could not open '%s': %s\n", path, sqlite3_errmsg(h)); + goto error; + } + + ret = cache_version_check(&have_version, h, want_version); + if (ret < 0) + { + DPRINTF(E_LOG, L_CACHE, "Could not check cache '%s' database version\n", name); + goto error; + } + + if (have_version > 0 && have_version < want_version) + { + DPRINTF(E_LOG, L_CACHE, "Database schema outdated, deleting cache '%s' v%d -> v%d\n", name, have_version, want_version); + + ret = cache_tables_drop(h, db_def, db_def_size); + if (ret < 0) + { + DPRINTF(E_LOG, L_CACHE, "Error deleting '%s' database tables\n", name); + goto error; + } + } + + if (have_version < want_version) + { + ret = cache_tables_create(h, want_version, db_def, db_def_size); + if (ret < 0) + { + DPRINTF(E_LOG, L_CACHE, "Could not create cache '%s' database tables\n", name); + goto error; + } + + DPRINTF(E_INFO, L_CACHE, "Cache '%s' database tables created\n", name); + } + + *hdl = h; + return 0; + + error: + sqlite3_close(h); + return -1; +} + +static int +cache_open(void) +{ + const char *directory; + const char *filename; + char *daap_db_path; + char *artwork_db_path; + char *xcode_db_path; + int ret; + + directory = cfg_getstr(cfg_getsec(cfg, "general"), "cache_dir"); + + CHECK_NULL(L_DB, filename = cfg_getstr(cfg_getsec(cfg, "general"), "cache_daap_filename")); + CHECK_NULL(L_DB, daap_db_path = safe_asprintf("%s%s", directory, filename)); + + CHECK_NULL(L_DB, filename = cfg_getstr(cfg_getsec(cfg, "general"), "cache_artwork_filename")); + CHECK_NULL(L_DB, artwork_db_path = safe_asprintf("%s%s", directory, filename)); + + CHECK_NULL(L_DB, filename = cfg_getstr(cfg_getsec(cfg, "general"), "cache_xcode_filename")); + CHECK_NULL(L_DB, xcode_db_path = safe_asprintf("%s%s", directory, filename)); + + ret = cache_open_one(&cache_daap_hdl, daap_db_path, "daap", CACHE_DAAP_VERSION, cache_daap_db_def, ARRAY_SIZE(cache_daap_db_def)); + if (ret < 0) + goto error; + + ret = cache_open_one(&cache_artwork_hdl, artwork_db_path, "artwork", CACHE_ARTWORK_VERSION, cache_artwork_db_def, ARRAY_SIZE(cache_artwork_db_def)); + if (ret < 0) + goto error; + + ret = cache_open_one(&cache_xcode_hdl, xcode_db_path, "xcode", CACHE_XCODE_VERSION, cache_xcode_db_def, ARRAY_SIZE(cache_xcode_db_def)); + if (ret < 0) + goto error; + + ret = cache_pragma_set(cache_artwork_hdl); + if (ret < 0) + goto error; + + DPRINTF(E_DBG, L_CACHE, "Cache opened\n"); + + free(daap_db_path); + free(artwork_db_path); + free(xcode_db_path); + return 0; + + error: + cache_close(); + free(daap_db_path); + free(artwork_db_path); + free(xcode_db_path); + return -1; +} + + /* Adds the reply (stored in evbuf) to the cache */ static int -cache_daap_reply_add(const char *query, struct evbuffer *evbuf) +cache_daap_reply_add(sqlite3 *hdl, const char *query, struct evbuffer *evbuf) { #define Q_TMPL "INSERT INTO replies (query, reply) VALUES (?, ?);" sqlite3_stmt *stmt; @@ -504,10 +566,10 @@ cache_daap_reply_add(const char *query, struct evbuffer *evbuf) datalen = evbuffer_get_length(evbuf); data = evbuffer_pullup(evbuf, -1); - ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + ret = sqlite3_prepare_v2(hdl, Q_TMPL, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error preparing query for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error preparing query for cache update: %s\n", sqlite3_errmsg(hdl)); return -1; } @@ -517,7 +579,7 @@ cache_daap_reply_add(const char *query, struct evbuffer *evbuf) ret = sqlite3_step(stmt); if (ret != SQLITE_DONE) { - DPRINTF(E_LOG, L_CACHE, "Error stepping query for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error stepping query for cache update: %s\n", sqlite3_errmsg(hdl)); sqlite3_finalize(stmt); return -1; } @@ -525,7 +587,7 @@ cache_daap_reply_add(const char *query, struct evbuffer *evbuf) ret = sqlite3_finalize(stmt); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error finalizing query for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error finalizing query for cache update: %s\n", sqlite3_errmsg(hdl)); return -1; } @@ -541,13 +603,12 @@ cache_daap_query_add(void *arg, int *retval) { #define Q_TMPL "INSERT OR REPLACE INTO queries (user_agent, is_remote, query, msec, timestamp) VALUES ('%q', %d, '%q', %d, %" PRIi64 ");" #define Q_CLEANUP "DELETE FROM queries WHERE id NOT IN (SELECT id FROM queries ORDER BY timestamp DESC LIMIT 20);" - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; struct timeval delay = { 60, 0 }; char *query; char *errmsg; int ret; - cmdarg = arg; if (!cmdarg->ua) { DPRINTF(E_LOG, L_CACHE, "Couldn't add slow query to cache, unknown user-agent\n"); @@ -573,7 +634,7 @@ cache_daap_query_add(void *arg, int *retval) goto error_add; } - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -589,7 +650,7 @@ cache_daap_query_add(void *arg, int *retval) free(cmdarg->query); // Limits the size of the cache to only contain replies for 20 most recent queries - ret = sqlite3_exec(g_db_hdl, Q_CLEANUP, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, Q_CLEANUP, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error cleaning up query list before update: %s\n", errmsg); @@ -624,22 +685,21 @@ static enum command_state cache_daap_query_get(void *arg, int *retval) { #define Q_TMPL "SELECT reply FROM replies WHERE query = ?;" - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; sqlite3_stmt *stmt; char *query; int datalen; int ret; - cmdarg = arg; query = cmdarg->query; remove_tag(query, "session-id"); remove_tag(query, "revision-number"); // Look in the DB - ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + ret = sqlite3_prepare_v2(cmdarg->hdl, Q_TMPL, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error preparing query for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error preparing query for cache update: %s\n", sqlite3_errmsg(cmdarg->hdl)); free(query); *retval = -1; return COMMAND_END; @@ -651,7 +711,7 @@ cache_daap_query_get(void *arg, int *retval) if (ret != SQLITE_ROW) { if (ret != SQLITE_DONE) - DPRINTF(E_LOG, L_CACHE, "Error stepping query for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error stepping query for cache update: %s\n", sqlite3_errmsg(cmdarg->hdl)); goto error_get; } @@ -672,7 +732,7 @@ cache_daap_query_get(void *arg, int *retval) ret = sqlite3_finalize(stmt); if (ret != SQLITE_OK) - DPRINTF(E_LOG, L_CACHE, "Error finalizing query for getting cache: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error finalizing query for getting cache: %s\n", sqlite3_errmsg(cmdarg->hdl)); DPRINTF(E_INFO, L_CACHE, "Cache hit: %s\n", query); @@ -691,7 +751,7 @@ cache_daap_query_get(void *arg, int *retval) /* Removes the query from the cache */ static int -cache_daap_query_delete(const int id) +cache_daap_query_delete(sqlite3 *hdl, const int id) { #define Q_TMPL "DELETE FROM queries WHERE id = %d;" char *query; @@ -700,7 +760,7 @@ cache_daap_query_delete(const int id) query = sqlite3_mprintf(Q_TMPL, id); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -720,6 +780,7 @@ cache_daap_query_delete(const int id) static void cache_daap_update_cb(int fd, short what, void *arg) { + sqlite3 *hdl = cache_daap_hdl; sqlite3_stmt *stmt; struct evbuffer *evbuf; struct evbuffer *gzbuf; @@ -727,15 +788,15 @@ cache_daap_update_cb(int fd, short what, void *arg) char *query; int ret; - if (g_suspended) + if (cache_is_suspended) { DPRINTF(E_DBG, L_CACHE, "Got a request to update DAAP cache while suspended\n"); return; } - DPRINTF(E_LOG, L_CACHE, "Beginning DAAP cache update\n"); + DPRINTF(E_INFO, L_CACHE, "Beginning DAAP cache update\n"); - ret = sqlite3_exec(g_db_hdl, "DELETE FROM replies;", NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, "DELETE FROM replies;", NULL, NULL, &errmsg); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error clearing reply cache before update: %s\n", errmsg); @@ -743,10 +804,10 @@ cache_daap_update_cb(int fd, short what, void *arg) return; } - ret = sqlite3_prepare_v2(g_db_hdl, "SELECT id, user_agent, is_remote, query FROM queries;", -1, &stmt, 0); + ret = sqlite3_prepare_v2(hdl, "SELECT id, user_agent, is_remote, query FROM queries;", -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error preparing for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error preparing for cache update: %s\n", sqlite3_errmsg(hdl)); return; } @@ -758,7 +819,7 @@ cache_daap_update_cb(int fd, short what, void *arg) if (!evbuf) { DPRINTF(E_LOG, L_CACHE, "Error building DAAP reply for query: %s\n", query); - cache_daap_query_delete(sqlite3_column_int(stmt, 0)); + cache_daap_query_delete(hdl, sqlite3_column_int(stmt, 0)); free(query); continue; @@ -768,7 +829,7 @@ cache_daap_update_cb(int fd, short what, void *arg) if (!gzbuf) { DPRINTF(E_LOG, L_CACHE, "Error gzipping DAAP reply for query: %s\n", query); - cache_daap_query_delete(sqlite3_column_int(stmt, 0)); + cache_daap_query_delete(hdl, sqlite3_column_int(stmt, 0)); free(query); evbuffer_free(evbuf); @@ -777,31 +838,31 @@ cache_daap_update_cb(int fd, short what, void *arg) evbuffer_free(evbuf); - cache_daap_reply_add(query, gzbuf); + cache_daap_reply_add(hdl, query, gzbuf); free(query); evbuffer_free(gzbuf); } if (ret != SQLITE_DONE) - DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(hdl)); sqlite3_finalize(stmt); - DPRINTF(E_LOG, L_CACHE, "DAAP cache updated\n"); + DPRINTF(E_INFO, L_CACHE, "DAAP cache updated\n"); } static enum command_state xcode_header_get(void *arg, int *retval) { -#define Q_TMPL "SELECT header FROM xcode_data WHERE length(header) > 0 AND id = ? AND format = ?;" +#define Q_TMPL "SELECT header FROM data WHERE length(header) > 0 AND id = ? AND format = ?;" struct cache_arg *cmdarg = arg; sqlite3_stmt *stmt = NULL; int ret; cmdarg->cached = 0; - ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + ret = sqlite3_prepare_v2(cmdarg->hdl, Q_TMPL, -1, &stmt, 0); if (ret != SQLITE_OK) goto error; @@ -828,7 +889,7 @@ xcode_header_get(void *arg, int *retval) return COMMAND_END; error: - DPRINTF(E_LOG, L_CACHE, "Database error getting prepared header from cache: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Database error getting prepared header from cache: %s\n", sqlite3_errmsg(cmdarg->hdl)); if (stmt) sqlite3_finalize(stmt); *retval = -1; @@ -837,9 +898,9 @@ xcode_header_get(void *arg, int *retval) } static int -xcode_add_entry(uint32_t id, uint32_t ts, const char *path) +xcode_add_entry(sqlite3 *hdl, uint32_t id, uint32_t ts, const char *path) { -#define Q_TMPL "INSERT OR REPLACE INTO xcode_files (id, time_modified, filepath) VALUES (%d, %d, '%q');" +#define Q_TMPL "INSERT OR REPLACE INTO files (id, time_modified, filepath) VALUES (%d, %d, '%q');" char *query; char *errmsg; int ret; @@ -848,7 +909,7 @@ xcode_add_entry(uint32_t id, uint32_t ts, const char *path) query = sqlite3_mprintf(Q_TMPL, id, ts, path); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -862,10 +923,10 @@ xcode_add_entry(uint32_t id, uint32_t ts, const char *path) } static int -xcode_del_entry(uint32_t id) +xcode_del_entry(sqlite3 *hdl, uint32_t id) { -#define Q_TMPL_FILES "DELETE FROM xcode_files WHERE id = %d;" -#define Q_TMPL_DATA "DELETE FROM xcode_data WHERE file_id = %d;" +#define Q_TMPL_FILES "DELETE FROM files WHERE id = %d;" +#define Q_TMPL_DATA "DELETE FROM data WHERE file_id = %d;" char query[256]; char *errmsg; int ret; @@ -873,16 +934,16 @@ xcode_del_entry(uint32_t id) DPRINTF(E_LOG, L_CACHE, "Deleting xcode file id %d\n", id); sqlite3_snprintf(sizeof(query), query, Q_TMPL_FILES, (int)id); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error deleting row from xcode_files: %s\n", errmsg); + DPRINTF(E_LOG, L_CACHE, "Error deleting row from xcode files: %s\n", errmsg); sqlite3_free(errmsg); return -1; } sqlite3_snprintf(sizeof(query), query, Q_TMPL_DATA, (int)id); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); if (ret != SQLITE_OK) { DPRINTF(E_LOG, L_CACHE, "Error deleting rows from xcode_data: %s\n", errmsg); @@ -908,7 +969,7 @@ xcode_del_entry(uint32_t id) * rows until: table end OR id is larger OR id is equal and time equal or newer */ static int -xcode_sync_with_files(void) +xcode_sync_with_files(sqlite3 *hdl) { sqlite3_stmt *stmt; struct cachelist *cachelist = NULL; @@ -921,10 +982,10 @@ xcode_sync_with_files(void) int i; int ret; - DPRINTF(E_LOG, L_CACHE, "SYNC START\n"); + DPRINTF(E_INFO, L_CACHE, "Beginning transcode sync\n"); // Both lists must be sorted by id, otherwise the compare below won't work - ret = sqlite3_prepare_v2(g_db_hdl, "SELECT id, time_modified FROM xcode_files ORDER BY id;", -1, &stmt, 0); + ret = sqlite3_prepare_v2(hdl, "SELECT id, time_modified FROM files ORDER BY id;", -1, &stmt, 0); if (ret != SQLITE_OK) goto error; @@ -953,7 +1014,7 @@ xcode_sync_with_files(void) if (ret != 0) // At end of files table (or error occured) { for (; i < cachelist_len; i++) - xcode_del_entry(cachelist[i].id); + xcode_del_entry(hdl, cachelist[i].id); break; } @@ -963,17 +1024,17 @@ xcode_sync_with_files(void) if (i == cachelist_len || cachelist[i].id > id) // At end of cache table or new file { - xcode_add_entry(id, ts, dbmfi.path); + xcode_add_entry(hdl, id, ts, dbmfi.path); } else if (cachelist[i].id < id) // Removed file { - xcode_del_entry(cachelist[i].id); + xcode_del_entry(hdl, cachelist[i].id); i++; } else if (cachelist[i].id == id && cachelist[i].ts < ts) // Modified file { - xcode_del_entry(cachelist[i].id); - xcode_add_entry(id, ts, dbmfi.path); + xcode_del_entry(hdl, cachelist[i].id); + xcode_add_entry(hdl, id, ts, dbmfi.path); i++; } else // Found in both tables and timestamp in cache table is adequate @@ -983,19 +1044,21 @@ xcode_sync_with_files(void) } db_query_end(&qp); + DPRINTF(E_INFO, L_CACHE, "Transcode sync completed\n"); + free(cachelist); return 0; error: - DPRINTF(E_LOG, L_CACHE, "Database error while processing xcode_files table\n"); + DPRINTF(E_LOG, L_CACHE, "Database error while processing xcode files table\n"); free(cachelist); return -1; } static int -xcode_prepare_header(const char *format, int id, const char *path) +xcode_prepare_header(sqlite3 *hdl, const char *format, int id, const char *path) { -#define Q_TMPL "INSERT INTO xcode_data (timestamp, file_id, format, header) VALUES (?, ?, ?, ?);" +#define Q_TMPL "INSERT INTO data (timestamp, file_id, format, header) VALUES (?, ?, ?, ?);" struct evbuffer *header = NULL; sqlite3_stmt *stmt = NULL; unsigned char *data = NULL; @@ -1016,10 +1079,10 @@ xcode_prepare_header(const char *format, int id, const char *path) datalen = 6; #endif - ret = sqlite3_prepare_v2(g_db_hdl, Q_TMPL, -1, &stmt, 0); + ret = sqlite3_prepare_v2(hdl, Q_TMPL, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error preparing xcode_data for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error preparing xcode_data for cache update: %s\n", sqlite3_errmsg(hdl)); goto error; } @@ -1031,7 +1094,7 @@ xcode_prepare_header(const char *format, int id, const char *path) ret = sqlite3_step(stmt); if (ret != SQLITE_DONE) { - DPRINTF(E_LOG, L_CACHE, "Error stepping xcode_data for cache update: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error stepping xcode_data for cache update: %s\n", sqlite3_errmsg(hdl)); goto error; } @@ -1050,37 +1113,37 @@ xcode_prepare_header(const char *format, int id, const char *path) } static int -xcode_prepare_headers(const char *format) +xcode_prepare_next_header(sqlite3 *hdl, const char *format) { -#define Q_TMPL "SELECT xf.id, xf.filepath, xd.id FROM xcode_files xf LEFT JOIN xcode_data xd ON xf.id = xd.file_id AND xd.format = '%q';" +#define Q_TMPL "SELECT f.id, f.filepath, d.id FROM files f LEFT JOIN data d ON f.id = d.file_id AND d.format = '%q' WHERE d.id IS NULL LIMIT 1;" sqlite3_stmt *stmt; char *query; const char *file_path; int file_id; - int data_id; int ret; query = sqlite3_mprintf(Q_TMPL, format); - ret = sqlite3_prepare_v2(g_db_hdl, query, -1, &stmt, 0); + ret = sqlite3_prepare_v2(hdl, query, -1, &stmt, 0); if (ret != SQLITE_OK) goto error; - while (sqlite3_step(stmt) == SQLITE_ROW) + ret = sqlite3_step(stmt); + if (ret != SQLITE_ROW) { - data_id = sqlite3_column_int(stmt, 2); - if (data_id > 0) - continue; // Already have a prepared header - - file_id = sqlite3_column_int(stmt, 0); - file_path = (const char *)sqlite3_column_text(stmt, 1); - - xcode_prepare_header(format, file_id, file_path); - + sqlite3_finalize(stmt); + sqlite3_free(query); + return -1; // All done } + + file_id = sqlite3_column_int(stmt, 0); + file_path = (const char *)sqlite3_column_text(stmt, 1); + + xcode_prepare_header(hdl, format, file_id, file_path); + sqlite3_finalize(stmt); sqlite3_free(query); - return 0; + return file_id; error: DPRINTF(E_LOG, L_CACHE, "Error occured while preparing headers\n"); @@ -1090,12 +1153,40 @@ xcode_prepare_headers(const char *format) } static void -cache_xcode_update_cb(int fd, short what, void *arg) +cache_xcode_prepare_cb(int fd, short what, void *arg) { - if (xcode_sync_with_files() < 0) + int ret; + + // Preparing headers can take very long, so we take one at a time, letting the + // event loop run in between + ret = xcode_prepare_next_header(cache_xcode_hdl, "mp4"); + if (ret < 0 || ret == cache_xcode_last_file) + { + DPRINTF(E_LOG, L_CACHE, "Header generation completed\n"); + return; + } + + // Used as failsafe to protect against infinite looping + cache_xcode_last_file = ret; + + if (!cache_is_initialized) return; - xcode_prepare_headers("mp4"); + event_active(cache_xcode_prepareev, 0, 0); +} + +static void +cache_xcode_update_cb(int fd, short what, void *arg) +{ + if (xcode_sync_with_files(cache_xcode_hdl) < 0) + return; + + if (!cache_is_initialized) + return; + + DPRINTF(E_LOG, L_CACHE, "Kicking off header generation\n"); + + event_active(cache_xcode_prepareev, 0, 0); } /* Sets off an update by activating the event. The delay is because we are low @@ -1106,10 +1197,11 @@ cache_database_update(void *arg, int *retval) { struct timeval delay_daap = { 10, 0 }; struct timeval delay_xcode = { 5, 0 }; -// const char *prefer_format = cfg_getstr(cfg_getsec(cfg, "library"), "prefer_format"); event_add(cache_daap_updateev, &delay_daap); +// TODO unlink or rename cache.db + // if (prefer_format && strcmp(prefer_format, "alac")) // TODO Ugly event_add(cache_xcode_updateev, &delay_xcode); @@ -1139,18 +1231,16 @@ cache_artwork_ping_impl(void *arg, int *retval) { #define Q_TMPL_PING "UPDATE artwork SET db_timestamp = %" PRIi64 " WHERE filepath = '%q' AND db_timestamp >= %" PRIi64 ";" #define Q_TMPL_DEL "DELETE FROM artwork WHERE filepath = '%q' AND db_timestamp < %" PRIi64 ";" - - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; char *query; char *errmsg; int ret; - cmdarg = arg; query = sqlite3_mprintf(Q_TMPL_PING, (int64_t)time(NULL), cmdarg->pathcopy, (int64_t)cmdarg->mtime); DPRINTF(E_DBG, L_CACHE, "Running query '%s'\n", query); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -1165,7 +1255,7 @@ cache_artwork_ping_impl(void *arg, int *retval) DPRINTF(E_DBG, L_CACHE, "Running query '%s'\n", query); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -1201,18 +1291,16 @@ static enum command_state cache_artwork_delete_by_path_impl(void *arg, int *retval) { #define Q_TMPL_DEL "DELETE FROM artwork WHERE filepath = '%q';" - - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; char *query; char *errmsg; int ret; - cmdarg = arg; query = sqlite3_mprintf(Q_TMPL_DEL, cmdarg->path); DPRINTF(E_DBG, L_CACHE, "Running query '%s'\n", query); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -1223,7 +1311,7 @@ cache_artwork_delete_by_path_impl(void *arg, int *retval) return COMMAND_END; } - DPRINTF(E_DBG, L_CACHE, "Deleted %d rows\n", sqlite3_changes(g_db_hdl)); + DPRINTF(E_DBG, L_CACHE, "Deleted %d rows\n", sqlite3_changes(cmdarg->hdl)); *retval = 0; return COMMAND_END; @@ -1242,17 +1330,16 @@ cache_artwork_purge_cruft_impl(void *arg, int *retval) { #define Q_TMPL "DELETE FROM artwork WHERE db_timestamp < %" PRIi64 ";" - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; char *query; char *errmsg; int ret; - cmdarg = arg; query = sqlite3_mprintf(Q_TMPL, (int64_t)cmdarg->mtime); DPRINTF(E_DBG, L_CACHE, "Running purge query '%s'\n", query); - ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg); + ret = sqlite3_exec(cmdarg->hdl, query, NULL, NULL, &errmsg); sqlite3_free(query); if (ret != SQLITE_OK) { @@ -1263,7 +1350,7 @@ cache_artwork_purge_cruft_impl(void *arg, int *retval) return COMMAND_END; } - DPRINTF(E_DBG, L_CACHE, "Purged %d rows\n", sqlite3_changes(g_db_hdl)); + DPRINTF(E_DBG, L_CACHE, "Purged %d rows\n", sqlite3_changes(cmdarg->hdl)); *retval = 0; return COMMAND_END; @@ -1285,20 +1372,19 @@ cache_artwork_purge_cruft_impl(void *arg, int *retval) static enum command_state cache_artwork_add_impl(void *arg, int *retval) { - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; sqlite3_stmt *stmt; char *query; uint8_t *data; int datalen; int ret; - cmdarg = arg; query = "INSERT INTO artwork (id, persistentid, max_w, max_h, format, filepath, db_timestamp, data, type) VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?);"; - ret = sqlite3_prepare_v2(g_db_hdl, query, -1, &stmt, 0); + ret = sqlite3_prepare_v2(cmdarg->hdl, query, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Could not prepare statement: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Could not prepare statement: %s\n", sqlite3_errmsg(cmdarg->hdl)); *retval = -1; return COMMAND_END; } @@ -1318,7 +1404,7 @@ cache_artwork_add_impl(void *arg, int *retval) ret = sqlite3_step(stmt); if (ret != SQLITE_DONE) { - DPRINTF(E_LOG, L_CACHE, "Error stepping query for artwork add: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error stepping query for artwork add: %s\n", sqlite3_errmsg(cmdarg->hdl)); sqlite3_finalize(stmt); *retval = -1; return COMMAND_END; @@ -1327,7 +1413,7 @@ cache_artwork_add_impl(void *arg, int *retval) ret = sqlite3_finalize(stmt); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error finalizing query for artwork add: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error finalizing query for artwork add: %s\n", sqlite3_errmsg(cmdarg->hdl)); *retval = -1; return COMMAND_END; } @@ -1355,13 +1441,12 @@ static enum command_state cache_artwork_get_impl(void *arg, int *retval) { #define Q_TMPL "SELECT a.format, a.data FROM artwork a WHERE a.type = %d AND a.persistentid = %" PRIi64 " AND a.max_w = %d AND a.max_h = %d;" - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; sqlite3_stmt *stmt; char *query; int datalen; int ret; - cmdarg = arg; query = sqlite3_mprintf(Q_TMPL, cmdarg->type, cmdarg->persistentid, cmdarg->max_w, cmdarg->max_h); if (!query) { @@ -1372,10 +1457,10 @@ cache_artwork_get_impl(void *arg, int *retval) DPRINTF(E_DBG, L_CACHE, "Running query '%s'\n", query); - ret = sqlite3_prepare_v2(g_db_hdl, query, -1, &stmt, 0); + ret = sqlite3_prepare_v2(cmdarg->hdl, query, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Could not prepare statement: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Could not prepare statement: %s\n", sqlite3_errmsg(cmdarg->hdl)); ret = -1; goto error_get; } @@ -1393,7 +1478,7 @@ cache_artwork_get_impl(void *arg, int *retval) else { ret = -1; - DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Could not step: %s\n", sqlite3_errmsg(cmdarg->hdl)); } goto error_get; @@ -1420,7 +1505,7 @@ cache_artwork_get_impl(void *arg, int *retval) ret = sqlite3_finalize(stmt); if (ret != SQLITE_OK) - DPRINTF(E_LOG, L_CACHE, "Error finalizing query for getting cache: %s\n", sqlite3_errmsg(g_db_hdl)); + DPRINTF(E_LOG, L_CACHE, "Error finalizing query for getting cache: %s\n", sqlite3_errmsg(cmdarg->hdl)); DPRINTF(E_DBG, L_CACHE, "Cache hit: %s\n", query); @@ -1441,16 +1526,14 @@ cache_artwork_get_impl(void *arg, int *retval) static enum command_state cache_artwork_stash_impl(void *arg, int *retval) { - struct cache_arg *cmdarg; - - cmdarg = arg; + struct cache_arg *cmdarg = arg; // Clear current stash - if (g_stash.path) + if (cache_stash.path) { - free(g_stash.path); - free(g_stash.data); - memset(&g_stash, 0, sizeof(struct stash)); + free(cache_stash.path); + free(cache_stash.data); + memset(&cache_stash, 0, sizeof(struct cache_artwork_stash)); } // If called with no evbuf then we are done, we just needed to clear the stash @@ -1460,49 +1543,48 @@ cache_artwork_stash_impl(void *arg, int *retval) return COMMAND_END; } - g_stash.size = evbuffer_get_length(cmdarg->evbuf); - g_stash.data = malloc(g_stash.size); - if (!g_stash.data) + cache_stash.size = evbuffer_get_length(cmdarg->evbuf); + cache_stash.data = malloc(cache_stash.size); + if (!cache_stash.data) { DPRINTF(E_LOG, L_CACHE, "Out of memory for artwork stash data\n"); *retval = -1; return COMMAND_END; } - g_stash.path = strdup(cmdarg->path); - if (!g_stash.path) + cache_stash.path = strdup(cmdarg->path); + if (!cache_stash.path) { DPRINTF(E_LOG, L_CACHE, "Out of memory for artwork stash path\n"); - free(g_stash.data); + free(cache_stash.data); *retval = -1; return COMMAND_END; } - g_stash.format = cmdarg->format; + cache_stash.format = cmdarg->format; - *retval = evbuffer_copyout(cmdarg->evbuf, g_stash.data, g_stash.size); + *retval = evbuffer_copyout(cmdarg->evbuf, cache_stash.data, cache_stash.size); return COMMAND_END; } static enum command_state cache_artwork_read_impl(void *arg, int *retval) { - struct cache_arg *cmdarg; + struct cache_arg *cmdarg = arg; - cmdarg = arg; cmdarg->format = 0; - if (!g_stash.path || !g_stash.data || (strcmp(g_stash.path, cmdarg->path) != 0)) + if (!cache_stash.path || !cache_stash.data || (strcmp(cache_stash.path, cmdarg->path) != 0)) { *retval = -1; return COMMAND_END; } - cmdarg->format = g_stash.format; + cmdarg->format = cache_stash.format; - DPRINTF(E_DBG, L_CACHE, "Stash hit (format %d, size %zu): %s\n", g_stash.format, g_stash.size, g_stash.path); + DPRINTF(E_DBG, L_CACHE, "Stash hit (format %d, size %zu): %s\n", cache_stash.format, cache_stash.size, cache_stash.path); - *retval = evbuffer_add(cmdarg->evbuf, g_stash.data, g_stash.size); + *retval = evbuffer_add(cmdarg->evbuf, cache_stash.data, cache_stash.size); return COMMAND_END; } @@ -1511,16 +1593,15 @@ cache(void *arg) { int ret; - ret = cache_create(); + ret = cache_open(); if (ret < 0) { DPRINTF(E_LOG, L_CACHE, "Error: Cache create failed. Cache will be disabled.\n"); pthread_exit(NULL); } - /* The thread needs a connection with the main db, so it can generate DAAP - * replies through httpd_daap.c and read changes from the files table - */ + // The thread needs a connection with the main db, so it can generate DAAP + // replies through httpd_daap.c and read changes from the files table ret = db_perthread_init(); if (ret < 0) { @@ -1530,16 +1611,28 @@ cache(void *arg) pthread_exit(NULL); } - g_initialized = 1; + CHECK_NULL(L_CACHE, cache_daap_updateev = evtimer_new(evbase_cache, cache_daap_update_cb, NULL)); + CHECK_NULL(L_CACHE, cache_xcode_updateev = evtimer_new(evbase_cache, cache_xcode_update_cb, NULL)); + CHECK_NULL(L_CACHE, cache_xcode_prepareev = evtimer_new(evbase_cache, cache_xcode_prepare_cb, NULL)); + + CHECK_ERR(L_CACHE, listener_add(cache_daap_listener_cb, LISTENER_DATABASE)); + + cache_is_initialized = 1; event_base_dispatch(evbase_cache); - if (g_initialized) + if (cache_is_initialized) { DPRINTF(E_LOG, L_CACHE, "Cache event loop terminated ahead of time!\n"); - g_initialized = 0; + cache_is_initialized = 0; } + listener_remove(cache_daap_listener_cb); + + event_free(cache_xcode_prepareev); + event_free(cache_xcode_updateev); + event_free(cache_daap_updateev); + db_perthread_deinit(); cache_close(); @@ -1560,13 +1653,13 @@ cache(void *arg) void cache_daap_suspend(void) { - g_suspended = 1; + cache_is_suspended = 1; } void cache_daap_resume(void) { - g_suspended = 0; + cache_is_suspended = 0; } int @@ -1574,9 +1667,10 @@ cache_daap_get(struct evbuffer *evbuf, const char *query) { struct cache_arg cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return -1; + cmdarg.hdl = cache_daap_hdl; cmdarg.query = strdup(query); cmdarg.evbuf = evbuf; @@ -1588,7 +1682,7 @@ cache_daap_add(const char *query, const char *ua, int is_remote, int msec) { struct cache_arg *cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return; cmdarg = calloc(1, sizeof(struct cache_arg)); @@ -1598,6 +1692,7 @@ cache_daap_add(const char *query, const char *ua, int is_remote, int msec) return; } + cmdarg->hdl = cache_daap_hdl; cmdarg->query = strdup(query); cmdarg->ua = strdup(ua); cmdarg->is_remote = is_remote; @@ -1607,9 +1702,9 @@ cache_daap_add(const char *query, const char *ua, int is_remote, int msec) } int -cache_daap_threshold(void) +cache_daap_threshold_get(void) { - return g_cfg_threshold; + return cache_daap_threshold; } @@ -1621,9 +1716,10 @@ cache_xcode_header_get(struct evbuffer *evbuf, int *cached, uint32_t id, const c struct cache_arg cmdarg; int ret; - if (!g_initialized) + if (!cache_is_initialized) return -1; + cmdarg.hdl = cache_xcode_hdl; cmdarg.evbuf = evbuf; cmdarg.id = id; cmdarg.header_format = format; @@ -1655,7 +1751,7 @@ cache_artwork_ping(const char *path, time_t mtime, int del) { struct cache_arg *cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return; cmdarg = calloc(1, sizeof(struct cache_arg)); @@ -1665,6 +1761,7 @@ cache_artwork_ping(const char *path, time_t mtime, int del) return; } + cmdarg->hdl = cache_artwork_hdl; cmdarg->pathcopy = strdup(path); cmdarg->mtime = mtime; cmdarg->del = del; @@ -1683,9 +1780,10 @@ cache_artwork_delete_by_path(const char *path) { struct cache_arg cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return -1; + cmdarg.hdl = cache_artwork_hdl; cmdarg.path = path; return commands_exec_sync(cmdbase, cache_artwork_delete_by_path_impl, NULL, &cmdarg); @@ -1702,9 +1800,10 @@ cache_artwork_purge_cruft(time_t ref) { struct cache_arg cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return -1; + cmdarg.hdl = cache_artwork_hdl; cmdarg.mtime = ref; return commands_exec_sync(cmdbase, cache_artwork_purge_cruft_impl, NULL, &cmdarg); @@ -1727,9 +1826,10 @@ cache_artwork_add(int type, int64_t persistentid, int max_w, int max_h, int form { struct cache_arg cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return -1; + cmdarg.hdl = cache_artwork_hdl; cmdarg.type = type; cmdarg.persistentid = persistentid; cmdarg.max_w = max_w; @@ -1761,13 +1861,14 @@ cache_artwork_get(int type, int64_t persistentid, int max_w, int max_h, int *cac struct cache_arg cmdarg; int ret; - if (!g_initialized) + if (!cache_is_initialized) { *cached = 0; *format = 0; return 0; } + cmdarg.hdl = cache_artwork_hdl; cmdarg.type = type; cmdarg.persistentid = persistentid; cmdarg.max_w = max_w; @@ -1795,7 +1896,7 @@ cache_artwork_stash(struct evbuffer *evbuf, const char *path, int format) { struct cache_arg cmdarg; - if (!g_initialized) + if (!cache_is_initialized) return -1; cmdarg.evbuf = evbuf; @@ -1819,7 +1920,7 @@ cache_artwork_read(struct evbuffer *evbuf, const char *path, int *format) struct cache_arg cmdarg; int ret; - if (!g_initialized) + if (!cache_is_initialized) return -1; cmdarg.evbuf = evbuf; @@ -1838,29 +1939,19 @@ cache_artwork_read(struct evbuffer *evbuf, const char *path, int *format) int cache_init(void) { - g_db_path = cfg_getstr(cfg_getsec(cfg, "general"), "cache_path"); - if (!g_db_path || (strlen(g_db_path) == 0)) - { - DPRINTF(E_LOG, L_CACHE, "Cache path invalid, disabling cache\n"); - return 0; - } - - g_cfg_threshold = cfg_getint(cfg_getsec(cfg, "general"), "cache_daap_threshold"); - if (g_cfg_threshold == 0) + cache_daap_threshold = cfg_getint(cfg_getsec(cfg, "general"), "cache_daap_threshold"); + if (cache_daap_threshold == 0) { DPRINTF(E_LOG, L_CACHE, "Cache threshold set to 0, disabling cache\n"); return 0; } CHECK_NULL(L_CACHE, evbase_cache = event_base_new()); - CHECK_NULL(L_CACHE, cache_daap_updateev = evtimer_new(evbase_cache, cache_daap_update_cb, NULL)); - CHECK_NULL(L_CACHE, cache_xcode_updateev = evtimer_new(evbase_cache, cache_xcode_update_cb, NULL)); CHECK_NULL(L_CACHE, cmdbase = commands_base_new(evbase_cache, NULL)); - CHECK_ERR(L_CACHE, listener_add(cache_daap_listener_cb, LISTENER_DATABASE)); CHECK_ERR(L_CACHE, pthread_create(&tid_cache, NULL, cache, NULL)); thread_setname(tid_cache, "cache"); - DPRINTF(E_INFO, L_CACHE, "cache thread init\n"); + DPRINTF(E_INFO, L_CACHE, "Cache thread init\n"); return 0; } @@ -1870,12 +1961,10 @@ cache_deinit(void) { int ret; - if (!g_initialized) + if (!cache_is_initialized) return; - g_initialized = 0; - - listener_remove(cache_daap_listener_cb); + cache_is_initialized = 0; commands_base_destroy(cmdbase); @@ -1886,7 +1975,5 @@ cache_deinit(void) return; } - // Free event base - event_free(cache_daap_updateev); event_base_free(evbase_cache); } diff --git a/src/cache.h b/src/cache.h index 036e09d7..5a0ccf4c 100644 --- a/src/cache.h +++ b/src/cache.h @@ -19,7 +19,7 @@ void cache_daap_add(const char *query, const char *ua, int is_remote, int msec); int -cache_daap_threshold(void); +cache_daap_threshold_get(void); /* --------------------------- Transcode cache API ------------------------- */ diff --git a/src/conffile.c b/src/conffile.c index c1c6dc05..15282d05 100644 --- a/src/conffile.c +++ b/src/conffile.c @@ -55,7 +55,7 @@ static cfg_opt_t sec_general[] = CFG_STR_LIST("trusted_networks", "{lan}", CFGF_NONE), CFG_BOOL("ipv6", cfg_false, CFGF_NONE), CFG_STR("bind_address", NULL, CFGF_NONE), - CFG_STR("cache_path", STATEDIR "/cache/" PACKAGE "/cache.db", CFGF_NONE), + CFG_STR("cache_dir", STATEDIR "/cache/" PACKAGE, CFGF_NONE), CFG_INT("cache_daap_threshold", 1000, CFGF_NONE), CFG_BOOL("speaker_autoselect", cfg_false, CFGF_NONE), #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) @@ -67,6 +67,9 @@ static cfg_opt_t sec_general[] = CFG_INT("db_pragma_cache_size", -1, CFGF_NONE), CFG_STR("db_pragma_journal_mode", NULL, CFGF_NONE), CFG_INT("db_pragma_synchronous", -1, CFGF_NONE), + CFG_STR("cache_daap_filename", "daap.db", CFGF_NONE), + CFG_STR("cache_artwork_filename", "artwork.db", CFGF_NONE), + CFG_STR("cache_xcode_filename", "xcode.db", CFGF_NONE), CFG_STR("allow_origin", "*", CFGF_NONE), CFG_STR("user_agent", PACKAGE_NAME "/" PACKAGE_VERSION, CFGF_NONE), CFG_BOOL("ssl_verifypeer", cfg_true, CFGF_NONE), @@ -312,6 +315,31 @@ cb_loglevel(cfg_t *config, cfg_opt_t *opt, const char *value, void *result) return 0; } +// Makes sure cache_dir ends with a slash +static int +sanitize_cache_dir(cfg_t *general) +{ + char *dir; + const char *s; + char *appended; + size_t len; + + dir = cfg_getstr(general, "cache_dir"); + len = strlen(dir); + + s = strrchr(dir, '/'); + if (s && (s + 1 == dir + len)) + return 0; + + appended = safe_asprintf("%s/", dir); + + cfg_setstr(general, "cache_dir", appended); + + free(appended); + + return 0; +} + static int conffile_expand_libname(cfg_t *lib) { @@ -425,7 +453,6 @@ conffile_expand_libname(cfg_t *lib) return 0; } - int conffile_load(char *file) { @@ -466,6 +493,14 @@ conffile_load(char *file) runas_uid = pw->pw_uid; runas_gid = pw->pw_gid; + ret = sanitize_cache_dir(cfg_getsec(cfg, "general")); + if (ret != 0) + { + DPRINTF(E_FATAL, L_CONF, "Invalid configuration of cache_dir\n"); + + goto out_fail; + } + lib = cfg_getsec(cfg, "library"); if (cfg_size(lib, "directories") == 0) diff --git a/src/db.c b/src/db.c index a4fe0ba5..12770c2e 100644 --- a/src/db.c +++ b/src/db.c @@ -6892,9 +6892,6 @@ db_open(void) int synchronous; int mmap_size; - if (!db_path) - return -1; - ret = sqlite3_open(db_path, &hdl); if (ret != SQLITE_OK) { @@ -7343,35 +7340,35 @@ db_init(void) db_path = cfg_getstr(cfg_getsec(cfg, "general"), "db_path"); db_rating_updates = cfg_getbool(cfg_getsec(cfg, "library"), "rating_updates"); - DPRINTF(E_LOG, L_DB, "Configured to use database file '%s'\n", db_path); + DPRINTF(E_INFO, L_DB, "Configured to use database file '%s'\n", db_path); ret = sqlite3_config(SQLITE_CONFIG_MULTITHREAD); if (ret != SQLITE_OK) { DPRINTF(E_FATAL, L_DB, "Could not switch SQLite3 to multithread mode\n"); DPRINTF(E_FATAL, L_DB, "Check that SQLite3 has been configured for thread-safe operations\n"); - return -1; + goto error; } ret = sqlite3_enable_shared_cache(1); if (ret != SQLITE_OK) { DPRINTF(E_FATAL, L_DB, "Could not enable SQLite3 shared-cache mode\n"); - return -1; + goto error; } ret = sqlite3_initialize(); if (ret != SQLITE_OK) { DPRINTF(E_FATAL, L_DB, "SQLite3 failed to initialize\n"); - return -1; + goto error; } ret = db_open(); if (ret < 0) { DPRINTF(E_FATAL, L_DB, "Could not open database\n"); - return -1; + goto error; } ret = db_check_version(); @@ -7380,7 +7377,7 @@ db_init(void) DPRINTF(E_FATAL, L_DB, "Database version check errored out, incompatible database\n"); db_perthread_deinit(); - return -1; + goto error; } else if (ret > 0) { @@ -7391,7 +7388,7 @@ db_init(void) { DPRINTF(E_FATAL, L_DB, "Could not create tables\n"); db_perthread_deinit(); - return -1; + goto error; } } @@ -7409,6 +7406,9 @@ db_init(void) rng_init(&shuffle_rng); return 0; + + error: + return -1; } void diff --git a/src/httpd_daap.c b/src/httpd_daap.c index 20f753cf..b2d50156 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -2276,7 +2276,7 @@ daap_request(struct httpd_request *hreq) DPRINTF(E_DBG, L_DAAP, "DAAP request handled in %d milliseconds\n", msec); - if (ret == DAAP_REPLY_OK && msec > cache_daap_threshold() && hreq->user_agent) + if (ret == DAAP_REPLY_OK && msec > cache_daap_threshold_get() && hreq->user_agent) cache_daap_add(hreq->uri, hreq->user_agent, ((struct daap_session *)hreq->extra_data)->is_remote, msec); daap_reply_send(hreq, ret); // hreq is deallocted From 9f719ca155692c4fc43986c7e66a0dd7c28aae76 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Sun, 7 Jan 2024 00:00:18 +0100 Subject: [PATCH 07/12] [player/jsonapi/db] Add interface to get and set an output format --- docs/json-api.md | 12 +++++++++--- src/db.c | 8 +++++--- src/db_init.c | 5 +++-- src/db_init.h | 2 +- src/db_upgrade.c | 25 +++++++++++++++++++++++++ src/httpd_jsonapi.c | 43 +++++++++++++++++++++++++++++++++++++++++++ src/player.c | 40 +++++++++++++++++++++++++++++++++++++--- src/player.h | 12 +++++++++++- 8 files changed, 134 insertions(+), 13 deletions(-) diff --git a/docs/json-api.md b/docs/json-api.md index 7107a29c..a988df25 100644 --- a/docs/json-api.md +++ b/docs/json-api.md @@ -341,6 +341,7 @@ GET /api/outputs | requires_auth | boolean | `true` if output requires authentication | | needs_auth_key | boolean | `true` if output requires an authorization key (device verification) | | volume | integer | Volume in percent (0 - 100) | +| format | string | Stream format | **Example** @@ -359,7 +360,8 @@ curl -X GET "http://localhost:3689/api/outputs" "has_password": false, "requires_auth": false, "needs_auth_key": false, - "volume": 0 + "volume": 0, + "format": "alac" }, { "id": "0", @@ -369,7 +371,8 @@ curl -X GET "http://localhost:3689/api/outputs" "has_password": false, "requires_auth": false, "needs_auth_key": false, - "volume": 19 + "volume": 19, + "format": "pcm" }, { "id": "100", @@ -379,7 +382,8 @@ curl -X GET "http://localhost:3689/api/outputs" "has_password": false, "requires_auth": false, "needs_auth_key": false, - "volume": 0 + "volume": 0, + "format": "pcm" } ] } @@ -448,6 +452,7 @@ curl -X GET "http://localhost:3689/api/outputs/0" "requires_auth": false, "needs_auth_key": false, "volume": 3 + "format": "pcm", } ``` @@ -474,6 +479,7 @@ PUT /api/outputs/{id} | selected | boolean | *(Optional)* `true` to enable and `false` to disable the output | | volume | integer | *(Optional)* Volume in percent (0 - 100) | | pin | string | *(Optional)* PIN for device verification | +| format | string | *(Optional)* Stream format | **Response** diff --git a/src/db.c b/src/db.c index 12770c2e..8a78c631 100644 --- a/src/db.c +++ b/src/db.c @@ -4787,10 +4787,10 @@ db_admin_delete(const char *key) int db_speaker_save(struct output_device *device) { -#define Q_TMPL "INSERT OR REPLACE INTO speakers (id, selected, volume, name, auth_key) VALUES (%" PRIi64 ", %d, %d, %Q, %Q);" +#define Q_TMPL "INSERT OR REPLACE INTO speakers (id, selected, volume, name, auth_key, format) VALUES (%" PRIi64 ", %d, %d, %Q, %Q, %d);" char *query; - query = sqlite3_mprintf(Q_TMPL, device->id, device->selected, device->volume, device->name, device->auth_key); + query = sqlite3_mprintf(Q_TMPL, device->id, device->selected, device->volume, device->name, device->auth_key, device->format); return db_query_run(query, 1, 0); #undef Q_TMPL @@ -4799,7 +4799,7 @@ db_speaker_save(struct output_device *device) int db_speaker_get(struct output_device *device, uint64_t id) { -#define Q_TMPL "SELECT s.selected, s.volume, s.name, s.auth_key FROM speakers s WHERE s.id = %" PRIi64 ";" +#define Q_TMPL "SELECT s.selected, s.volume, s.name, s.auth_key, s.format FROM speakers s WHERE s.id = %" PRIi64 ";" sqlite3_stmt *stmt; char *query; int ret; @@ -4841,6 +4841,8 @@ db_speaker_get(struct output_device *device, uint64_t id) free(device->auth_key); device->auth_key = safe_strdup((char *)sqlite3_column_text(stmt, 3)); + device->format = sqlite3_column_int(stmt, 4); + #ifdef DB_PROFILE while (db_blocking_step(stmt) == SQLITE_ROW) ; /* EMPTY */ diff --git a/src/db_init.c b/src/db_init.c index 1a406e1d..db1492a2 100644 --- a/src/db_init.c +++ b/src/db_init.c @@ -151,8 +151,9 @@ " id INTEGER PRIMARY KEY NOT NULL," \ " selected INTEGER NOT NULL," \ " volume INTEGER NOT NULL," \ - " name VARCHAR(255) DEFAULT NULL," \ - " auth_key VARCHAR(2048) DEFAULT NULL" \ + " name VARCHAR(255) DEFAULT NULL," \ + " auth_key VARCHAR(2048) DEFAULT NULL," \ + " format INTEGER DEFAULT 0" \ ");" #define T_INOTIFY \ diff --git a/src/db_init.h b/src/db_init.h index aaf6ba1b..20052f1d 100644 --- a/src/db_init.h +++ b/src/db_init.h @@ -26,7 +26,7 @@ * is a major upgrade. In other words minor version upgrades permit downgrading * the server after the database was upgraded. */ #define SCHEMA_VERSION_MAJOR 22 -#define SCHEMA_VERSION_MINOR 1 +#define SCHEMA_VERSION_MINOR 2 int db_init_indices(sqlite3 *hdl); diff --git a/src/db_upgrade.c b/src/db_upgrade.c index 90fff3b8..efc1c74a 100644 --- a/src/db_upgrade.c +++ b/src/db_upgrade.c @@ -1246,6 +1246,24 @@ static const struct db_upgrade_query db_upgrade_v2201_queries[] = }; +/* ---------------------------- 22.01 -> 22.02 ------------------------------ */ + +#define U_v2202_ALTER_SPEAKERS_ADD_FORMAT \ + "ALTER TABLE speakers ADD COLUMN format INTEGER DEFAULT 0;" + +#define U_v2202_SCVER_MAJOR \ + "UPDATE admin SET value = '22' WHERE key = 'schema_version_major';" +#define U_v2202_SCVER_MINOR \ + "UPDATE admin SET value = '02' WHERE key = 'schema_version_minor';" + +static const struct db_upgrade_query db_upgrade_v2202_queries[] = + { + { U_v2202_ALTER_SPEAKERS_ADD_FORMAT, "alter table speakers add column format" }, + + { U_v2202_SCVER_MAJOR, "set schema_version_major to 22" }, + { U_v2202_SCVER_MINOR, "set schema_version_minor to 02" }, + }; + /* -------------------------- Main upgrade handler -------------------------- */ @@ -1464,6 +1482,13 @@ db_upgrade(sqlite3 *hdl, int db_ver) if (ret < 0) return -1; + /* FALLTHROUGH */ + + case 2201: + ret = db_generic_upgrade(hdl, db_upgrade_v2202_queries, ARRAY_SIZE(db_upgrade_v2202_queries)); + if (ret < 0) + return -1; + /* Last case statement is the only one that ends with a break statement! */ break; diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index 24d3e7da..5e9de857 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -1522,6 +1522,40 @@ struct outputs_param int output_volume; }; +static enum player_format +plformat_from_string(const char *format) +{ + if (strcmp(format, "pcm") == 0) + return PLAYER_FORMAT_PCM; + if (strcmp(format, "wav") == 0) + return PLAYER_FORMAT_WAV; + if (strcmp(format, "mp3") == 0) + return PLAYER_FORMAT_MP3; + if (strcmp(format, "alac") == 0) + return PLAYER_FORMAT_ALAC; + if (strcmp(format, "opus") == 0) + return PLAYER_FORMAT_OPUS; + + return PLAYER_FORMAT_UNKNOWN; +} + +static const char * +plformat_to_string(enum player_format format) +{ + if (format == PLAYER_FORMAT_PCM) + return "pcm"; + if (format == PLAYER_FORMAT_WAV) + return "wav"; + if (format == PLAYER_FORMAT_MP3) + return "mp3"; + if (format == PLAYER_FORMAT_ALAC) + return "alac"; + if (format == PLAYER_FORMAT_OPUS) + return "opus"; + + return "unknown"; +} + static json_object * speaker_to_json(struct player_speaker_info *spk) { @@ -1539,6 +1573,7 @@ speaker_to_json(struct player_speaker_info *spk) json_object_object_add(output, "requires_auth", json_object_new_boolean(spk->requires_auth)); json_object_object_add(output, "needs_auth_key", json_object_new_boolean(spk->needs_auth_key)); json_object_object_add(output, "volume", json_object_new_int(spk->absvol)); + json_object_object_add(output, "format", json_object_new_string(plformat_to_string(spk->format))); return output; } @@ -1602,6 +1637,7 @@ jsonapi_reply_outputs_put_byid(struct httpd_request *hreq) bool selected; int volume; const char *pin; + const char *format; int ret; ret = safe_atou64(hreq->path_parts[2], &output_id); @@ -1644,6 +1680,13 @@ jsonapi_reply_outputs_put_byid(struct httpd_request *hreq) ret = player_speaker_authorize(output_id, pin); } + if (ret == 0 && jparse_contains_key(request, "format", json_type_string)) + { + format = jparse_str_from_obj(request, "format"); + if (format) + ret = player_speaker_format_set(output_id, plformat_from_string(format)); + } + jparse_free(request); if (ret < 0) diff --git a/src/player.c b/src/player.c index 7147a0e9..96975dca 100644 --- a/src/player.c +++ b/src/player.c @@ -2526,6 +2526,7 @@ device_to_speaker_info(struct player_speaker_info *spk, struct output_device *de spk->output_type[sizeof(spk->output_type) - 1] = '\0'; spk->relvol = device->relvol; spk->absvol = device->volume; + spk->format = device->format; spk->selected = OUTPUTS_DEVICE_DISPLAY_SELECTED(device); @@ -2730,6 +2731,8 @@ speaker_prevent_playback_set(void *arg, int *retval) struct speaker_attr_param *param = arg; struct output_device *device; + *retval = -1; + device = outputs_device_get(param->spk_id); if (!device) return COMMAND_END; @@ -2778,6 +2781,8 @@ speaker_busy_set(void *arg, int *retval) struct speaker_attr_param *param = arg; struct output_device *device; + *retval = -1; + device = outputs_device_get(param->spk_id); if (!device) return COMMAND_END; @@ -2803,6 +2808,27 @@ speaker_busy_set(void *arg, int *retval) return COMMAND_END; } +static enum command_state +speaker_format_set(void *arg, int *retval) +{ + struct speaker_attr_param *param = arg; + struct output_device *device; + + *retval = -1; + + if (param->format == PLAYER_FORMAT_UNKNOWN) + return COMMAND_END; + + device = outputs_device_get(param->spk_id); + if (!device) + return COMMAND_END; + + device->format = param->format; + + *retval = 0; + return COMMAND_END; +} + // Attempts to reactivate a speaker that has failed. That includes restarting // playback if it was stopped. static enum command_state @@ -3466,14 +3492,22 @@ int player_speaker_authorize(uint64_t id, const char *pin) { struct speaker_attr_param param; - int ret; param.spk_id = id; param.pin = pin; - ret = commands_exec_sync(cmdbase, speaker_authorize, speaker_generic_bh, ¶m); + return commands_exec_sync(cmdbase, speaker_authorize, speaker_generic_bh, ¶m); +} - return ret; +int +player_speaker_format_set(uint64_t id, enum player_format format) +{ + struct speaker_attr_param param; + + param.spk_id = id; + param.format = format; + + return commands_exec_sync(cmdbase, speaker_format_set, NULL, ¶m); } int diff --git a/src/player.h b/src/player.h index 602a13e0..8aed0aae 100644 --- a/src/player.h +++ b/src/player.h @@ -29,7 +29,12 @@ enum player_seek_mode { }; enum player_format { - PLAYER_FORMAT_MP3, + PLAYER_FORMAT_UNKNOWN = -1, + PLAYER_FORMAT_PCM = 0, + PLAYER_FORMAT_WAV = 1, + PLAYER_FORMAT_MP3 = 2, + PLAYER_FORMAT_ALAC = 3, + PLAYER_FORMAT_OPUS = 4, }; struct player_speaker_info { @@ -40,6 +45,8 @@ struct player_speaker_info { int relvol; int absvol; + enum player_format format; + bool selected; bool has_password; bool requires_auth; @@ -122,6 +129,9 @@ player_speaker_resurrect(void *arg); int player_speaker_authorize(uint64_t id, const char *pin); +int +player_speaker_format_set(uint64_t id, enum player_format format); + int player_streaming_register(int *audio_fd, int *metadata_fd, enum player_format format, struct media_quality quality); From 62b42ce354d687f23dde6901327556c26637a87d Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Sun, 7 Jan 2024 23:12:03 +0100 Subject: [PATCH 08/12] [misc/player] Introduce output ability to announce supported formats Also introduce default output format and selected device format, should the user want another format. As part of this, change enum player_format in player.h to enum media_format in misc.h so that it is akin to struct media_quality. Modify json API to support this. --- docs/json-api.md | 11 ++++++--- src/cache.c | 40 +++++++++++++++++++++++++++------ src/cache.h | 4 ++++ src/db.c | 4 ++-- src/httpd.c | 50 +++++++++++++++++++++++++++++++++-------- src/httpd.h | 11 --------- src/httpd_jsonapi.c | 50 +++++++++++------------------------------ src/httpd_streaming.c | 4 ++-- src/misc.c | 34 ++++++++++++++++++++++++++++ src/misc.h | 21 +++++++++++++++++ src/outputs.h | 6 ++++- src/outputs/airplay.c | 1 + src/outputs/alsa.c | 1 + src/outputs/cast.c | 1 + src/outputs/fifo.c | 1 + src/outputs/pulse.c | 1 + src/outputs/raop.c | 1 + src/outputs/rcp.c | 2 ++ src/outputs/streaming.c | 16 ++++++------- src/player.c | 38 +++++++++++++++++++++---------- src/player.h | 16 ++++--------- 21 files changed, 209 insertions(+), 104 deletions(-) diff --git a/docs/json-api.md b/docs/json-api.md index a988df25..8de4674d 100644 --- a/docs/json-api.md +++ b/docs/json-api.md @@ -342,6 +342,7 @@ GET /api/outputs | needs_auth_key | boolean | `true` if output requires an authorization key (device verification) | | volume | integer | Volume in percent (0 - 100) | | format | string | Stream format | +| supported_formats | array | Array of formats supported by output | **Example** @@ -361,7 +362,8 @@ curl -X GET "http://localhost:3689/api/outputs" "requires_auth": false, "needs_auth_key": false, "volume": 0, - "format": "alac" + "format": "alac", + "supported_formats": [ "alac" ] }, { "id": "0", @@ -372,7 +374,8 @@ curl -X GET "http://localhost:3689/api/outputs" "requires_auth": false, "needs_auth_key": false, "volume": 19, - "format": "pcm" + "format": "pcm", + "supported_formats": [ "pcm" ] }, { "id": "100", @@ -383,7 +386,8 @@ curl -X GET "http://localhost:3689/api/outputs" "requires_auth": false, "needs_auth_key": false, "volume": 0, - "format": "pcm" + "format": "pcm", + "supported_formats": [ "pcm" ] } ] } @@ -453,6 +457,7 @@ curl -X GET "http://localhost:3689/api/outputs/0" "needs_auth_key": false, "volume": 3 "format": "pcm", + "supported_formats": [ "pcm" ] } ``` diff --git a/src/cache.c b/src/cache.c index f61a7001..9dc62067 100644 --- a/src/cache.c +++ b/src/cache.c @@ -37,8 +37,9 @@ #include "conffile.h" #include "logger.h" -#include "httpd.h" +#include "httpd.h" // TODO get rid of this, only used for httpd_gzip_deflate #include "httpd_daap.h" +#include "transcode.h" #include "db.h" #include "cache.h" #include "listener.h" @@ -189,6 +190,7 @@ static struct cache_db_def cache_artwork_db_def[] = { static sqlite3 *cache_xcode_hdl; static struct event *cache_xcode_updateev; static struct event *cache_xcode_prepareev; +static bool cache_xcode_is_enabled; static int cache_xcode_last_file; static struct cache_db_def cache_xcode_db_def[] = { DB_DEF_ADMIN, @@ -897,6 +899,20 @@ xcode_header_get(void *arg, int *retval) #undef Q_TMPL } +static enum command_state +xcode_toggle(void *arg, int *retval) +{ + bool *enable = arg; + + cache_xcode_is_enabled = *enable; + + if (cache_xcode_is_enabled) + event_active(cache_xcode_updateev, 0, 0); + + *retval = 0; + return COMMAND_END; +} + static int xcode_add_entry(sqlite3 *hdl, uint32_t id, uint32_t ts, const char *path) { @@ -982,8 +998,6 @@ xcode_sync_with_files(sqlite3 *hdl) int i; int ret; - DPRINTF(E_INFO, L_CACHE, "Beginning transcode sync\n"); - // Both lists must be sorted by id, otherwise the compare below won't work ret = sqlite3_prepare_v2(hdl, "SELECT id, time_modified FROM files ORDER BY id;", -1, &stmt, 0); if (ret != SQLITE_OK) @@ -1044,8 +1058,6 @@ xcode_sync_with_files(sqlite3 *hdl) } db_query_end(&qp); - DPRINTF(E_INFO, L_CACHE, "Transcode sync completed\n"); - free(cachelist); return 0; @@ -1068,7 +1080,12 @@ xcode_prepare_header(sqlite3 *hdl, const char *format, int id, const char *path) DPRINTF(E_DBG, L_CACHE, "Preparing %s header for '%s' (file id %d)\n", format, path, id); #if 1 - ret = httpd_prepare_header(&header, format, path); // Proceed even if error, we also cache that + if (strcmp(format, "mp4") == 0) + ret = transcode_prepare_header(&header, XCODE_MP4_ALAC, path); + else + ret = -1; + + // Proceed even if error, we also cache that if (ret == 0) { datalen = evbuffer_get_length(header); @@ -1202,7 +1219,7 @@ cache_database_update(void *arg, int *retval) // TODO unlink or rename cache.db -// if (prefer_format && strcmp(prefer_format, "alac")) // TODO Ugly + if (cache_xcode_is_enabled) event_add(cache_xcode_updateev, &delay_xcode); *retval = 0; @@ -1731,6 +1748,15 @@ cache_xcode_header_get(struct evbuffer *evbuf, int *cached, uint32_t id, const c return ret; } +int +cache_xcode_toggle(bool enable) +{ + if (!cache_is_initialized) + return -1; + + return commands_exec_sync(cmdbase, xcode_toggle, NULL, &enable); +} + /* ---------------------------- Artwork cache API -------------------------- */ diff --git a/src/cache.h b/src/cache.h index 5a0ccf4c..aefe5502 100644 --- a/src/cache.h +++ b/src/cache.h @@ -27,6 +27,10 @@ cache_daap_threshold_get(void); int cache_xcode_header_get(struct evbuffer *evbuf, int *cached, uint32_t id, const char *format); +int +cache_xcode_toggle(bool enable); + + /* ---------------------------- Artwork cache API -------------------------- */ #define CACHE_ARTWORK_GROUP 0 diff --git a/src/db.c b/src/db.c index 8a78c631..8b23af16 100644 --- a/src/db.c +++ b/src/db.c @@ -4790,7 +4790,7 @@ db_speaker_save(struct output_device *device) #define Q_TMPL "INSERT OR REPLACE INTO speakers (id, selected, volume, name, auth_key, format) VALUES (%" PRIi64 ", %d, %d, %Q, %Q, %d);" char *query; - query = sqlite3_mprintf(Q_TMPL, device->id, device->selected, device->volume, device->name, device->auth_key, device->format); + query = sqlite3_mprintf(Q_TMPL, device->id, device->selected, device->volume, device->name, device->auth_key, device->selected_format); return db_query_run(query, 1, 0); #undef Q_TMPL @@ -4841,7 +4841,7 @@ db_speaker_get(struct output_device *device, uint64_t id) free(device->auth_key); device->auth_key = safe_strdup((char *)sqlite3_column_text(stmt, 3)); - device->format = sqlite3_column_int(stmt, 4); + device->selected_format = sqlite3_column_int(stmt, 4); #ifdef DB_PROFILE while (db_blocking_step(stmt) == SQLITE_ROW) diff --git a/src/httpd.c b/src/httpd.c index 07527ed7..b54d8dba 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -49,6 +49,8 @@ #include "httpd_internal.h" #include "transcode.h" #include "cache.h" +#include "listener.h" +#include "player.h" #ifdef LASTFM # include "lastfm.h" #endif @@ -905,6 +907,39 @@ stream_fail_cb(void *arg) } +/* -------------------------- SPEAKER/CACHE HANDLING ------------------------ */ + +// Thread: player (must not block) +static void +speaker_enum_cb(struct player_speaker_info *spk, void *arg) +{ + bool *want_mp4 = arg; + + *want_mp4 = *want_mp4 || (spk->format == MEDIA_FORMAT_ALAC && strcmp(spk->output_type, "RCP/SoundBridge") == 0); +} + +// Thread: worker +static void +speaker_update_handler_cb(void *arg) +{ + const char *prefer_format = cfg_getstr(cfg_getsec(cfg, "library"), "prefer_format"); + bool want_mp4; + + want_mp4 = (prefer_format && strcmp(prefer_format, "alac")); + if (!want_mp4) + player_speaker_enumerate(speaker_enum_cb, &want_mp4); + + cache_xcode_toggle(want_mp4); +} + +// Thread: player (must not block) +static void +httpd_speaker_update_handler(short event_mask) +{ + worker_execute(speaker_update_handler_cb, NULL, 0, 0); +} + + /* ---------------------------- REQUEST CALLBACKS --------------------------- */ // Worker thread, invoked by request_cb() below @@ -1220,15 +1255,6 @@ httpd_gzip_deflate(struct evbuffer *in) return NULL; } -int -httpd_prepare_header(struct evbuffer **header, const char *format, const char *path) -{ - if (strcmp(format, "mp4") == 0) - return transcode_prepare_header(header, XCODE_MP4_ALAC, path); - else - return -1; -} - // The httpd_send functions below can be called from a worker thread (with // hreq->is_async) or directly from the httpd thread. In the former case, they // will command sending from the httpd thread, since it is not safe to access @@ -1543,6 +1569,10 @@ httpd_init(const char *webroot) goto error; } + // We need to know about speaker format changes so we can ask the cache to + // start preparing headers for mp4/alac if selected + listener_add(httpd_speaker_update_handler, LISTENER_SPEAKER); + return 0; error: @@ -1554,6 +1584,8 @@ httpd_init(const char *webroot) void httpd_deinit(void) { + listener_remove(httpd_speaker_update_handler); + // Give modules a chance to hang up connections nicely modules_deinit(); diff --git a/src/httpd.h b/src/httpd.h index 0fed50c6..8926f707 100644 --- a/src/httpd.h +++ b/src/httpd.h @@ -13,17 +13,6 @@ struct evbuffer * httpd_gzip_deflate(struct evbuffer *in); -/* - * Passthrough to transcode, which will create a transcoded file header for path - * - * @out header Newly created evbuffer with the header - * @in format Which format caller wants a header for - * @in path Path to the file - * @return 0 if ok, otherwise -1 - */ -int -httpd_prepare_header(struct evbuffer **header, const char *format, const char *path); - int httpd_init(const char *webroot); diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index 5e9de857..09984541 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -1522,48 +1522,23 @@ struct outputs_param int output_volume; }; -static enum player_format -plformat_from_string(const char *format) -{ - if (strcmp(format, "pcm") == 0) - return PLAYER_FORMAT_PCM; - if (strcmp(format, "wav") == 0) - return PLAYER_FORMAT_WAV; - if (strcmp(format, "mp3") == 0) - return PLAYER_FORMAT_MP3; - if (strcmp(format, "alac") == 0) - return PLAYER_FORMAT_ALAC; - if (strcmp(format, "opus") == 0) - return PLAYER_FORMAT_OPUS; - - return PLAYER_FORMAT_UNKNOWN; -} - -static const char * -plformat_to_string(enum player_format format) -{ - if (format == PLAYER_FORMAT_PCM) - return "pcm"; - if (format == PLAYER_FORMAT_WAV) - return "wav"; - if (format == PLAYER_FORMAT_MP3) - return "mp3"; - if (format == PLAYER_FORMAT_ALAC) - return "alac"; - if (format == PLAYER_FORMAT_OPUS) - return "opus"; - - return "unknown"; -} - static json_object * speaker_to_json(struct player_speaker_info *spk) { json_object *output; + json_object *supported_formats; char output_id[21]; + enum media_format format; output = json_object_new_object(); + supported_formats = json_object_new_array(); + for (format = MEDIA_FORMAT_FIRST; format <= MEDIA_FORMAT_LAST; format = MEDIA_FORMAT_NEXT(format)) + { + if (format & spk->supported_formats) + json_object_array_add(supported_formats, json_object_new_string(media_format_to_string(format))); + } + snprintf(output_id, sizeof(output_id), "%" PRIu64, spk->id); json_object_object_add(output, "id", json_object_new_string(output_id)); json_object_object_add(output, "name", json_object_new_string(spk->name)); @@ -1573,7 +1548,8 @@ speaker_to_json(struct player_speaker_info *spk) json_object_object_add(output, "requires_auth", json_object_new_boolean(spk->requires_auth)); json_object_object_add(output, "needs_auth_key", json_object_new_boolean(spk->needs_auth_key)); json_object_object_add(output, "volume", json_object_new_int(spk->absvol)); - json_object_object_add(output, "format", json_object_new_string(plformat_to_string(spk->format))); + json_object_object_add(output, "format", json_object_new_string(media_format_to_string(spk->format))); + json_object_object_add(output, "supported_formats", supported_formats); return output; } @@ -1684,13 +1660,13 @@ jsonapi_reply_outputs_put_byid(struct httpd_request *hreq) { format = jparse_str_from_obj(request, "format"); if (format) - ret = player_speaker_format_set(output_id, plformat_from_string(format)); + ret = player_speaker_format_set(output_id, media_format_from_string(format)); } jparse_free(request); if (ret < 0) - return HTTP_INTERNAL; + return HTTP_BADREQUEST; return HTTP_NOCONTENT; } diff --git a/src/httpd_streaming.c b/src/httpd_streaming.c index 7590ca25..92d170b3 100644 --- a/src/httpd_streaming.c +++ b/src/httpd_streaming.c @@ -228,7 +228,7 @@ session_free(struct streaming_session *session) } static struct streaming_session * -session_new(struct httpd_request *hreq, bool icy_is_requested, enum player_format format, struct media_quality quality) +session_new(struct httpd_request *hreq, bool icy_is_requested, enum media_format format, struct media_quality quality) { struct streaming_session *session; int audio_fd; @@ -279,7 +279,7 @@ streaming_mp3_handler(struct httpd_request *hreq) httpd_header_add(hreq->out_headers, "icy-metaint", buf); } - session = session_new(hreq, icy_is_requested, PLAYER_FORMAT_MP3, streaming_default_quality); + session = session_new(hreq, icy_is_requested, MEDIA_FORMAT_MP3, streaming_default_quality); if (!session) return -1; // Error sent by caller diff --git a/src/misc.c b/src/misc.c index f662f3d1..921d990d 100644 --- a/src/misc.c +++ b/src/misc.c @@ -1731,6 +1731,40 @@ quality_is_equal(struct media_quality *a, struct media_quality *b) return (a->sample_rate == b->sample_rate && a->bits_per_sample == b->bits_per_sample && a->channels == b->channels && a->bit_rate == b->bit_rate); } +enum media_format +media_format_from_string(const char *s) +{ + if (strcmp(s, "pcm") == 0) + return MEDIA_FORMAT_PCM; + if (strcmp(s, "wav") == 0) + return MEDIA_FORMAT_WAV; + if (strcmp(s, "mp3") == 0) + return MEDIA_FORMAT_MP3; + if (strcmp(s, "alac") == 0) + return MEDIA_FORMAT_ALAC; + if (strcmp(s, "opus") == 0) + return MEDIA_FORMAT_OPUS; + + return MEDIA_FORMAT_UNKNOWN; +} + +const char * +media_format_to_string(enum media_format format) +{ + if (format == MEDIA_FORMAT_PCM) + return "pcm"; + if (format == MEDIA_FORMAT_WAV) + return "wav"; + if (format == MEDIA_FORMAT_MP3) + return "mp3"; + if (format == MEDIA_FORMAT_ALAC) + return "alac"; + if (format == MEDIA_FORMAT_OPUS) + return "opus"; + + return "unknown"; +} + /* -------------------------- Misc utility functions ------------------------ */ diff --git a/src/misc.h b/src/misc.h index e3d1b7d9..fe5615a5 100644 --- a/src/misc.h +++ b/src/misc.h @@ -272,6 +272,21 @@ timespec_reltoabs(struct timespec relative); /* ------------------------------- Media quality ---------------------------- */ +// Bit flags for the sake of outputs announcing what they support +enum media_format { + MEDIA_FORMAT_UNKNOWN = 0, + MEDIA_FORMAT_PCM = (1 << 0), + MEDIA_FORMAT_WAV = (1 << 1), + MEDIA_FORMAT_MP3 = (1 << 2), + MEDIA_FORMAT_ALAC = (1 << 3), + MEDIA_FORMAT_OPUS = (1 << 4), +}; + +// For iteration +#define MEDIA_FORMAT_FIRST MEDIA_FORMAT_PCM +#define MEDIA_FORMAT_LAST MEDIA_FORMAT_OPUS +#define MEDIA_FORMAT_NEXT(f) (f << 1) + // Remember to adjust quality_is_equal() if adding elements struct media_quality { int sample_rate; @@ -283,6 +298,12 @@ struct media_quality { bool quality_is_equal(struct media_quality *a, struct media_quality *b); +enum media_format +media_format_from_string(const char *s); + +const char * +media_format_to_string(enum media_format format); + /* -------------------------- Misc utility functions ------------------------ */ diff --git a/src/outputs.h b/src/outputs.h index 61d380c1..eb1def82 100644 --- a/src/outputs.h +++ b/src/outputs.h @@ -134,7 +134,11 @@ struct output_device // Quality of audio output struct media_quality quality; - int format; + + // selected_format only set (not UNKNOWN) in case of active user selection + enum media_format selected_format; + enum media_format default_format; + uint32_t supported_formats; // Address char *v4_address; diff --git a/src/outputs/airplay.c b/src/outputs/airplay.c index 7e6b3cf7..c18b4625 100644 --- a/src/outputs/airplay.c +++ b/src/outputs/airplay.c @@ -3737,6 +3737,7 @@ airplay_device_cb(const char *name, const char *type, const char *domain, const rd->type = OUTPUT_TYPE_AIRPLAY; rd->type_name = outputs_name(rd->type); rd->extra_device_info = re; + rd->supported_formats = MEDIA_FORMAT_ALAC; if (port < 0) { diff --git a/src/outputs/alsa.c b/src/outputs/alsa.c index 8e27027c..37fdb902 100644 --- a/src/outputs/alsa.c +++ b/src/outputs/alsa.c @@ -1374,6 +1374,7 @@ alsa_device_add(cfg_t* cfg_audio, int id) device->type = OUTPUT_TYPE_ALSA; device->type_name = outputs_name(device->type); device->extra_device_info = ae; + device->supported_formats = MEDIA_FORMAT_PCM; // The audio section will have no title, so there we get the value from the // "card" option diff --git a/src/outputs/cast.c b/src/outputs/cast.c index b6cc5ffb..80cc1494 100644 --- a/src/outputs/cast.c +++ b/src/outputs/cast.c @@ -1778,6 +1778,7 @@ cast_device_cb(const char *name, const char *type, const char *domain, const cha device->name = strdup(name); device->type = OUTPUT_TYPE_CAST; device->type_name = outputs_name(device->type); + device->supported_formats = MEDIA_FORMAT_OPUS; if (port < 0) { diff --git a/src/outputs/fifo.c b/src/outputs/fifo.c index b87510b6..ae2f6e41 100644 --- a/src/outputs/fifo.c +++ b/src/outputs/fifo.c @@ -491,6 +491,7 @@ fifo_init(void) device->type_name = outputs_name(device->type); device->has_video = 0; device->extra_device_info = path; + device->supported_formats = MEDIA_FORMAT_PCM; DPRINTF(E_INFO, L_FIFO, "Adding fifo output device '%s' with path '%s'\n", nickname, path); player_device_add(device); diff --git a/src/outputs/pulse.c b/src/outputs/pulse.c index cd3ea95e..c110870f 100644 --- a/src/outputs/pulse.c +++ b/src/outputs/pulse.c @@ -436,6 +436,7 @@ sinklist_cb(pa_context *ctx, const pa_sink_info *info, int eol, void *userdata) device->type = OUTPUT_TYPE_PULSE; device->type_name = outputs_name(device->type); device->extra_device_info = strdup(info->name); + device->supported_formats = MEDIA_FORMAT_PCM; player_device_add(device); } diff --git a/src/outputs/raop.c b/src/outputs/raop.c index 20436aaa..f37a6b7d 100644 --- a/src/outputs/raop.c +++ b/src/outputs/raop.c @@ -4230,6 +4230,7 @@ raop_device_cb(const char *name, const char *type, const char *domain, const cha rd->type = OUTPUT_TYPE_RAOP; rd->type_name = outputs_name(rd->type); rd->extra_device_info = re; + rd->supported_formats = MEDIA_FORMAT_ALAC; if (port < 0) { diff --git a/src/outputs/rcp.c b/src/outputs/rcp.c index ebae9da7..212a6897 100644 --- a/src/outputs/rcp.c +++ b/src/outputs/rcp.c @@ -1295,6 +1295,8 @@ rcp_mdns_device_cb(const char *name, const char *type, const char *domain, const device->name = strdup(name); device->type = OUTPUT_TYPE_RCP; device->type_name = outputs_name(device->type); + device->default_format = MEDIA_FORMAT_WAV; + device->supported_formats = MEDIA_FORMAT_WAV | MEDIA_FORMAT_MP3 | MEDIA_FORMAT_ALAC; if (port < 0 || !address) { diff --git a/src/outputs/streaming.c b/src/outputs/streaming.c index ecb5314b..11c43234 100644 --- a/src/outputs/streaming.c +++ b/src/outputs/streaming.c @@ -65,7 +65,7 @@ struct streaming_wanted struct pipepair audio[WANTED_PIPES_MAX]; struct pipepair metadata[WANTED_PIPES_MAX]; - enum player_format format; + enum media_format format; struct media_quality quality; struct evbuffer *audio_in; @@ -113,7 +113,7 @@ extern struct event_base *evbase_player; /* ------------------------------- Helpers ---------------------------------- */ static struct encode_ctx * -encoder_setup(enum player_format format, struct media_quality *quality) +encoder_setup(enum media_format format, struct media_quality *quality) { struct transcode_encode_setup_args encode_args = { .profile = XCODE_MP3, .quality = quality }; struct encode_ctx *encode_ctx = NULL; @@ -132,7 +132,7 @@ encoder_setup(enum player_format format, struct media_quality *quality) goto out; } - if (format == PLAYER_FORMAT_MP3) + if (format == MEDIA_FORMAT_MP3) encode_ctx = transcode_encode_setup(encode_args); if (!encode_ctx) @@ -217,7 +217,7 @@ pipe_index_find_byreadfd(struct pipepair *p, int readfd) } static struct streaming_wanted * -wanted_new(enum player_format format, struct media_quality quality) +wanted_new(enum media_format format, struct media_quality quality) { struct streaming_wanted *w; @@ -277,7 +277,7 @@ wanted_remove(struct streaming_wanted **wanted, struct streaming_wanted *remove) } static struct streaming_wanted * -wanted_add(struct streaming_wanted **wanted, enum player_format format, struct media_quality quality) +wanted_add(struct streaming_wanted **wanted, enum media_format format, struct media_quality quality) { struct streaming_wanted *w; @@ -289,7 +289,7 @@ wanted_add(struct streaming_wanted **wanted, enum player_format format, struct m } static struct streaming_wanted * -wanted_find_byformat(struct streaming_wanted *wanted, enum player_format format, struct media_quality quality) +wanted_find_byformat(struct streaming_wanted *wanted, enum media_format format, struct media_quality quality) { struct streaming_wanted *w; @@ -623,9 +623,9 @@ streaming_start(struct output_device *device, int callback_id) int ret; pthread_mutex_lock(&streaming_wanted_lck); - w = wanted_find_byformat(streaming.wanted, device->format, device->quality); + w = wanted_find_byformat(streaming.wanted, device->selected_format, device->quality); if (!w) - w = wanted_add(&streaming.wanted, device->format, device->quality); + w = wanted_add(&streaming.wanted, device->selected_format, device->quality); ret = wanted_session_add(&device->audio_fd, &device->metadata_fd, w); if (ret < 0) goto error; diff --git a/src/player.c b/src/player.c index 96975dca..3e11a96c 100644 --- a/src/player.c +++ b/src/player.c @@ -152,7 +152,7 @@ struct speaker_attr_param bool busy; struct media_quality quality; - enum player_format format; + enum media_format format; int audio_fd; int metadata_fd; @@ -2526,7 +2526,15 @@ device_to_speaker_info(struct player_speaker_info *spk, struct output_device *de spk->output_type[sizeof(spk->output_type) - 1] = '\0'; spk->relvol = device->relvol; spk->absvol = device->volume; - spk->format = device->format; + + spk->supported_formats = device->supported_formats; + // Devices supporting more than one format should at least have default_format set + if (device->selected_format != MEDIA_FORMAT_UNKNOWN) + spk->format = device->selected_format; + else if (device->default_format != MEDIA_FORMAT_UNKNOWN) + spk->format = device->default_format; + else + spk->format = device->supported_formats; spk->selected = OUTPUTS_DEVICE_DISPLAY_SELECTED(device); @@ -2814,19 +2822,25 @@ speaker_format_set(void *arg, int *retval) struct speaker_attr_param *param = arg; struct output_device *device; - *retval = -1; - - if (param->format == PLAYER_FORMAT_UNKNOWN) - return COMMAND_END; + if (param->format == MEDIA_FORMAT_UNKNOWN) + goto error; device = outputs_device_get(param->spk_id); if (!device) - return COMMAND_END; + goto error; - device->format = param->format; + if (!(param->format & device->supported_formats)) + goto error; + + device->selected_format = param->format; *retval = 0; return COMMAND_END; + + error: + DPRINTF(E_LOG, L_PLAYER, "Error setting format '%s', device unknown or format unsupported\n", media_format_to_string(param->format)); + *retval = -1; + return COMMAND_END; } // Attempts to reactivate a speaker that has failed. That includes restarting @@ -2933,7 +2947,7 @@ streaming_register(void *arg, int *retval) .type_name = "streaming", .name = "streaming", .quality = param->quality, - .format = param->format, + .selected_format = param->format, }; *retval = outputs_device_start(&device, NULL, false); @@ -3500,18 +3514,18 @@ player_speaker_authorize(uint64_t id, const char *pin) } int -player_speaker_format_set(uint64_t id, enum player_format format) +player_speaker_format_set(uint64_t id, enum media_format format) { struct speaker_attr_param param; param.spk_id = id; param.format = format; - return commands_exec_sync(cmdbase, speaker_format_set, NULL, ¶m); + return commands_exec_sync(cmdbase, speaker_format_set, speaker_generic_bh, ¶m); } int -player_streaming_register(int *audio_fd, int *metadata_fd, enum player_format format, struct media_quality quality) +player_streaming_register(int *audio_fd, int *metadata_fd, enum media_format format, struct media_quality quality) { struct speaker_attr_param param; int ret; diff --git a/src/player.h b/src/player.h index 8aed0aae..3f7867a8 100644 --- a/src/player.h +++ b/src/player.h @@ -28,15 +28,6 @@ enum player_seek_mode { PLAYER_SEEK_RELATIVE = 2, }; -enum player_format { - PLAYER_FORMAT_UNKNOWN = -1, - PLAYER_FORMAT_PCM = 0, - PLAYER_FORMAT_WAV = 1, - PLAYER_FORMAT_MP3 = 2, - PLAYER_FORMAT_ALAC = 3, - PLAYER_FORMAT_OPUS = 4, -}; - struct player_speaker_info { uint64_t id; uint32_t active_remote; @@ -45,7 +36,8 @@ struct player_speaker_info { int relvol; int absvol; - enum player_format format; + enum media_format format; + uint32_t supported_formats; bool selected; bool has_password; @@ -130,10 +122,10 @@ int player_speaker_authorize(uint64_t id, const char *pin); int -player_speaker_format_set(uint64_t id, enum player_format format); +player_speaker_format_set(uint64_t id, enum media_format format); int -player_streaming_register(int *audio_fd, int *metadata_fd, enum player_format format, struct media_quality quality); +player_streaming_register(int *audio_fd, int *metadata_fd, enum media_format format, struct media_quality quality); int player_streaming_deregister(int id); From ff2d0b4ab12a0ec619e4c996469aa8d3c5c61e29 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Wed, 10 Jan 2024 21:20:31 +0100 Subject: [PATCH 09/12] [httpd] Check speaker settings to determine DAAP/RSP stream format --- src/httpd.c | 31 ++++++++++++++++++++++++++++++- src/httpd_daap.c | 8 ++++---- src/httpd_internal.h | 3 +++ src/httpd_libevhttp.c | 6 ++++++ src/httpd_rsp.c | 10 +++++----- src/player.c | 39 +++++++++++++++++++++++++++++++++++++++ src/player.h | 3 +++ src/transcode.c | 2 +- src/transcode.h | 2 +- 9 files changed, 92 insertions(+), 12 deletions(-) diff --git a/src/httpd.c b/src/httpd.c index b54d8dba..2be9daae 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -1069,7 +1069,7 @@ httpd_stream_file(struct httpd_request *hreq, int id) } param = httpd_header_find(hreq->in_headers, "Accept-Codecs"); - profile = transcode_needed(hreq->user_agent, param, mfi->codectype); + profile = httpd_xcode_profile_get(hreq->user_agent, hreq->peer_address, param, mfi->codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_HTTPD, "Could not serve '%s' to client, unable to determine output format\n", mfi->path); @@ -1188,6 +1188,35 @@ httpd_stream_file(struct httpd_request *hreq, int id) free_mfi(mfi, 0); } +// Returns enum transcode_profile, but is just declared with int so we don't +// need to include transcode.h in httpd_internal.h +int +httpd_xcode_profile_get(const char *user_agent, const char *address, const char *accept_codecs, const char *codec) +{ + enum transcode_profile profile; + struct player_speaker_info spk; + int ret; + + profile = transcode_needed(user_agent, accept_codecs, codec); + if (profile == XCODE_NONE) + return profile; + + // A Roku Soundbridge may also be RCP device/speaker for which the user may + // have set a prefered streaming format + ret = player_speaker_get_byaddress(&spk, address); + if (ret < 0) + return profile; + + if (spk.format == MEDIA_FORMAT_WAV) + return XCODE_WAV; + if (spk.format == MEDIA_FORMAT_MP3) + return XCODE_MP3; + if (spk.format == MEDIA_FORMAT_ALAC) + return XCODE_MP4_ALAC; + + return profile; +} + struct evbuffer * httpd_gzip_deflate(struct evbuffer *in) { diff --git a/src/httpd_daap.c b/src/httpd_daap.c index b2d50156..5924e9e6 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -1146,7 +1146,7 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) const struct dmap_field **meta = NULL; struct sort_ctx *sctx; const char *param; - const char *client_codecs; + const char *accept_codecs; const char *tag; size_t len; enum transcode_profile profile; @@ -1216,10 +1216,10 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) goto error; } - client_codecs = NULL; + accept_codecs = NULL; if (!s->is_remote && hreq->in_headers) { - client_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); + accept_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); } nsongs = 0; @@ -1229,7 +1229,7 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) // Not sure if the is_remote path is really needed. Note that if you // change the below you might need to do the same in rsp_reply_playlist() - profile = s->is_remote ? XCODE_WAV : transcode_needed(hreq->user_agent, client_codecs, dbmfi.codectype); + profile = s->is_remote ? XCODE_WAV : httpd_xcode_profile_get(hreq->user_agent, hreq->peer_address, accept_codecs, dbmfi.codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_DAAP, "Cannot transcode '%s', codec type is unknown\n", dbmfi.fname); diff --git a/src/httpd_internal.h b/src/httpd_internal.h index 28262572..b9a0ad93 100644 --- a/src/httpd_internal.h +++ b/src/httpd_internal.h @@ -206,6 +206,9 @@ struct httpd_request { void httpd_stream_file(struct httpd_request *hreq, int id); +int +httpd_xcode_profile_get(const char *user_agent, const char *address, const char *accept_codecs, const char *codec); + void httpd_request_handler_set(struct httpd_request *hreq); diff --git a/src/httpd_libevhttp.c b/src/httpd_libevhttp.c index 071034fc..b5ac8d5b 100644 --- a/src/httpd_libevhttp.c +++ b/src/httpd_libevhttp.c @@ -525,6 +525,7 @@ httpd_backend_output_buffer_get(httpd_backend *backend) int httpd_backend_peer_get(const char **addr, uint16_t *port, httpd_backend *backend, httpd_backend_data *backend_data) { +#define IPV4_MAPPED_IPV6_PREFIX "::ffff:" httpd_connection *conn = evhttp_request_get_connection(backend); if (!conn) return -1; @@ -534,6 +535,11 @@ httpd_backend_peer_get(const char **addr, uint16_t *port, httpd_backend *backend #else evhttp_connection_get_peer(conn, (char **)addr, port); #endif + + // Just use the pure ipv4 address if it's mapped + if (strncmp(*addr, IPV4_MAPPED_IPV6_PREFIX, strlen(IPV4_MAPPED_IPV6_PREFIX)) == 0) + *addr += strlen(IPV4_MAPPED_IPV6_PREFIX); + return 0; } diff --git a/src/httpd_rsp.c b/src/httpd_rsp.c index 4e53b067..1a78ab75 100644 --- a/src/httpd_rsp.c +++ b/src/httpd_rsp.c @@ -415,7 +415,7 @@ rsp_reply_db(struct httpd_request *hreq) } static int -item_add(xml_node *parent, struct query_params *qp, const char *user_agent, const char *client_codecs, int mode) +item_add(xml_node *parent, struct query_params *qp, const char *user_agent, const char *address, const char *accept_codecs, int mode) { struct media_quality quality = { 0 }; struct db_media_file_info dbmfi; @@ -432,7 +432,7 @@ item_add(xml_node *parent, struct query_params *qp, const char *user_agent, cons if (ret != 0) return ret; - profile = transcode_needed(user_agent, client_codecs, dbmfi.codectype); + profile = httpd_xcode_profile_get(user_agent, address, accept_codecs, dbmfi.codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_DAAP, "Cannot transcode '%s', codec type is unknown\n", dbmfi.fname); @@ -484,7 +484,7 @@ rsp_reply_playlist(struct httpd_request *hreq) { struct query_params qp; const char *param; - const char *client_codecs; + const char *accept_codecs; xml_node *response; xml_node *items; int mode; @@ -493,7 +493,7 @@ rsp_reply_playlist(struct httpd_request *hreq) memset(&qp, 0, sizeof(struct query_params)); - client_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); + accept_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); ret = safe_atoi32(hreq->path_parts[2], &qp.id); if (ret < 0) @@ -555,7 +555,7 @@ rsp_reply_playlist(struct httpd_request *hreq) items = xml_new_node(response, "items", NULL); do { - ret = item_add(items, &qp, hreq->user_agent, client_codecs, mode); + ret = item_add(items, &qp, hreq->user_agent, hreq->peer_address, accept_codecs, mode); } while (ret == 0); diff --git a/src/player.c b/src/player.c index 3e11a96c..6101f6f0 100644 --- a/src/player.c +++ b/src/player.c @@ -164,6 +164,7 @@ struct speaker_get_param { uint64_t spk_id; uint32_t active_remote; + const char *address; struct player_speaker_info *spk_info; }; @@ -2606,6 +2607,31 @@ speaker_get_byactiveremote(void *arg, int *retval) return COMMAND_END; } +static enum command_state +speaker_get_byaddress(void *arg, int *retval) +{ + struct speaker_get_param *spk_param = arg; + struct output_device *device; + bool match_v4; + bool match_v6; + + for (device = outputs_list(); device; device = device->next) + { + match_v4 = device->v4_address && (strcmp(spk_param->address, device->v4_address) == 0); + match_v6 = device->v6_address && (strcmp(spk_param->address, device->v6_address) == 0); + if (match_v4 || match_v6) + { + device_to_speaker_info(spk_param->spk_info, device); + *retval = 0; + return COMMAND_END; + } + } + + // No output device found with matching id + *retval = -1; + return COMMAND_END; +} + static enum command_state speaker_set(void *arg, int *retval) { @@ -3444,6 +3470,19 @@ player_speaker_get_byactiveremote(struct player_speaker_info *spk, uint32_t acti return ret; } +int +player_speaker_get_byaddress(struct player_speaker_info *spk, const char *address) +{ + struct speaker_get_param param; + int ret; + + param.address = address; + param.spk_info = spk; + + ret = commands_exec_sync(cmdbase, speaker_get_byaddress, NULL, ¶m); + return ret; +} + int player_speaker_enable(uint64_t id) { diff --git a/src/player.h b/src/player.h index 3f7867a8..2292c5ea 100644 --- a/src/player.h +++ b/src/player.h @@ -103,6 +103,9 @@ player_speaker_get_byid(struct player_speaker_info *spk, uint64_t id); int player_speaker_get_byactiveremote(struct player_speaker_info *spk, uint32_t active_remote); +int +player_speaker_get_byaddress(struct player_speaker_info *spk, const char *address); + int player_speaker_enable(uint64_t id); diff --git a/src/transcode.c b/src/transcode.c index 37c1d595..a58326e4 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -2097,7 +2097,7 @@ transcode_decode_setup_raw(enum transcode_profile profile, struct media_quality } enum transcode_profile -transcode_needed(const char *user_agent, const char *client_codecs, char *file_codectype) +transcode_needed(const char *user_agent, const char *client_codecs, const char *file_codectype) { const char *codectype; const char *prefer_format; diff --git a/src/transcode.h b/src/transcode.h index c9a7f372..916461e8 100644 --- a/src/transcode.h +++ b/src/transcode.h @@ -112,7 +112,7 @@ struct decode_ctx * transcode_decode_setup_raw(enum transcode_profile profile, struct media_quality *quality); enum transcode_profile -transcode_needed(const char *user_agent, const char *client_codecs, char *file_codectype); +transcode_needed(const char *user_agent, const char *client_codecs, const char *file_codectype); // Cleaning up void From 2d9200fcdfda60ffcd3e248063104b44fde8ca7f Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Thu, 11 Jan 2024 22:40:32 +0100 Subject: [PATCH 10/12] [cache] Try to fix cache thread blocking the rest of the server --- src/cache.c | 37 ++++++++++++++++++++++++++++--------- src/httpd.c | 2 ++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/cache.c b/src/cache.c index 9dc62067..b78219fd 100644 --- a/src/cache.c +++ b/src/cache.c @@ -41,6 +41,7 @@ #include "httpd_daap.h" #include "transcode.h" #include "db.h" +#include "worker.h" #include "cache.h" #include "listener.h" #include "commands.h" @@ -191,6 +192,7 @@ static sqlite3 *cache_xcode_hdl; static struct event *cache_xcode_updateev; static struct event *cache_xcode_prepareev; static bool cache_xcode_is_enabled; +static bool cache_xcode_prepare_is_running; static int cache_xcode_last_file; static struct cache_db_def cache_xcode_db_def[] = { DB_DEF_ADMIN, @@ -899,6 +901,15 @@ xcode_header_get(void *arg, int *retval) #undef Q_TMPL } +static void +xcode_trigger(void) +{ + struct timeval delay_xcode = { 5, 0 }; + + if (cache_xcode_is_enabled) + event_add(cache_xcode_updateev, &delay_xcode); +} + static enum command_state xcode_toggle(void *arg, int *retval) { @@ -906,8 +917,7 @@ xcode_toggle(void *arg, int *retval) cache_xcode_is_enabled = *enable; - if (cache_xcode_is_enabled) - event_active(cache_xcode_updateev, 0, 0); + xcode_trigger(); *retval = 0; return COMMAND_END; @@ -1130,7 +1140,7 @@ xcode_prepare_header(sqlite3 *hdl, const char *format, int id, const char *path) } static int -xcode_prepare_next_header(sqlite3 *hdl, const char *format) +xcode_prepare_next_header_impl(sqlite3 *hdl, const char *format) { #define Q_TMPL "SELECT f.id, f.filepath, d.id FROM files f LEFT JOIN data d ON f.id = d.file_id AND d.format = '%q' WHERE d.id IS NULL LIMIT 1;" sqlite3_stmt *stmt; @@ -1170,16 +1180,17 @@ xcode_prepare_next_header(sqlite3 *hdl, const char *format) } static void -cache_xcode_prepare_cb(int fd, short what, void *arg) +xcode_prepare_next_header(void *arg) { int ret; // Preparing headers can take very long, so we take one at a time, letting the // event loop run in between - ret = xcode_prepare_next_header(cache_xcode_hdl, "mp4"); + ret = xcode_prepare_next_header_impl(cache_xcode_hdl, "mp4"); if (ret < 0 || ret == cache_xcode_last_file) { DPRINTF(E_LOG, L_CACHE, "Header generation completed\n"); + cache_xcode_prepare_is_running = false; return; } @@ -1192,17 +1203,24 @@ cache_xcode_prepare_cb(int fd, short what, void *arg) event_active(cache_xcode_prepareev, 0, 0); } +static void +cache_xcode_prepare_cb(int fd, short what, void *arg) +{ + worker_execute(xcode_prepare_next_header, NULL, 0, 0); +} + static void cache_xcode_update_cb(int fd, short what, void *arg) { if (xcode_sync_with_files(cache_xcode_hdl) < 0) return; - if (!cache_is_initialized) + if (!cache_is_initialized || cache_xcode_prepare_is_running) return; DPRINTF(E_LOG, L_CACHE, "Kicking off header generation\n"); + cache_xcode_prepare_is_running = true; event_active(cache_xcode_prepareev, 0, 0); } @@ -1213,14 +1231,12 @@ static enum command_state cache_database_update(void *arg, int *retval) { struct timeval delay_daap = { 10, 0 }; - struct timeval delay_xcode = { 5, 0 }; event_add(cache_daap_updateev, &delay_daap); // TODO unlink or rename cache.db - if (cache_xcode_is_enabled) - event_add(cache_xcode_updateev, &delay_xcode); + xcode_trigger(); *retval = 0; return COMMAND_END; @@ -1631,6 +1647,7 @@ cache(void *arg) CHECK_NULL(L_CACHE, cache_daap_updateev = evtimer_new(evbase_cache, cache_daap_update_cb, NULL)); CHECK_NULL(L_CACHE, cache_xcode_updateev = evtimer_new(evbase_cache, cache_xcode_update_cb, NULL)); CHECK_NULL(L_CACHE, cache_xcode_prepareev = evtimer_new(evbase_cache, cache_xcode_prepare_cb, NULL)); + CHECK_ERR(L_CACHE, event_priority_set(cache_xcode_prepareev, 0)); CHECK_ERR(L_CACHE, listener_add(cache_daap_listener_cb, LISTENER_DATABASE)); @@ -1973,7 +1990,9 @@ cache_init(void) } CHECK_NULL(L_CACHE, evbase_cache = event_base_new()); + CHECK_ERR(L_CACHE, event_base_priority_init(evbase_cache, 8)); CHECK_NULL(L_CACHE, cmdbase = commands_base_new(evbase_cache, NULL)); + CHECK_ERR(L_CACHE, pthread_create(&tid_cache, NULL, cache, NULL)); thread_setname(tid_cache, "cache"); diff --git a/src/httpd.c b/src/httpd.c index 2be9daae..af36a089 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -1201,6 +1201,8 @@ httpd_xcode_profile_get(const char *user_agent, const char *address, const char if (profile == XCODE_NONE) return profile; + DPRINTF(E_DBG, L_HTTPD, "Checking if client '%s' is a speaker\n", address); + // A Roku Soundbridge may also be RCP device/speaker for which the user may // have set a prefered streaming format ret = player_speaker_get_byaddress(&spk, address); From 088c393dd6c1640395f66629275dc8e9f99134cc Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Sun, 14 Jan 2024 21:38:04 +0100 Subject: [PATCH 11/12] [httpd/cache] Misc fixing up --- src/cache.c | 7 +++++-- src/httpd.c | 24 ++++++++++++------------ src/httpd_daap.c | 8 +++++++- src/httpd_internal.h | 2 +- src/httpd_rsp.c | 11 ++++++++--- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/cache.c b/src/cache.c index b78219fd..8b45fc31 100644 --- a/src/cache.c +++ b/src/cache.c @@ -859,7 +859,7 @@ cache_daap_update_cb(int fd, short what, void *arg) static enum command_state xcode_header_get(void *arg, int *retval) { -#define Q_TMPL "SELECT header FROM data WHERE length(header) > 0 AND id = ? AND format = ?;" +#define Q_TMPL "SELECT header FROM data WHERE length(header) > 0 AND file_id = ? AND format = ?;" struct cache_arg *cmdarg = arg; sqlite3_stmt *stmt = NULL; int ret; @@ -915,10 +915,13 @@ xcode_toggle(void *arg, int *retval) { bool *enable = arg; - cache_xcode_is_enabled = *enable; + if (*enable == cache_xcode_is_enabled) + goto end; + cache_xcode_is_enabled = *enable; xcode_trigger(); + end: *retval = 0; return COMMAND_END; } diff --git a/src/httpd.c b/src/httpd.c index af36a089..c0c64c8a 100644 --- a/src/httpd.c +++ b/src/httpd.c @@ -925,7 +925,7 @@ speaker_update_handler_cb(void *arg) const char *prefer_format = cfg_getstr(cfg_getsec(cfg, "library"), "prefer_format"); bool want_mp4; - want_mp4 = (prefer_format && strcmp(prefer_format, "alac")); + want_mp4 = (prefer_format && (strcmp(prefer_format, "alac") == 0)); if (!want_mp4) player_speaker_enumerate(speaker_enum_cb, &want_mp4); @@ -1009,6 +1009,7 @@ httpd_stream_file(struct httpd_request *hreq, int id) struct media_file_info *mfi = NULL; struct stream_ctx *st = NULL; enum transcode_profile profile; + enum transcode_profile spk_profile; const char *param; const char *param_end; const char *ctype; @@ -1069,7 +1070,7 @@ httpd_stream_file(struct httpd_request *hreq, int id) } param = httpd_header_find(hreq->in_headers, "Accept-Codecs"); - profile = httpd_xcode_profile_get(hreq->user_agent, hreq->peer_address, param, mfi->codectype); + profile = transcode_needed(hreq->user_agent, param, mfi->codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_HTTPD, "Could not serve '%s' to client, unable to determine output format\n", mfi->path); @@ -1082,6 +1083,10 @@ httpd_stream_file(struct httpd_request *hreq, int id) { DPRINTF(E_INFO, L_HTTPD, "Preparing to transcode %s\n", mfi->path); + spk_profile = httpd_xcode_profile_get(hreq); + if (spk_profile != XCODE_NONE) + profile = spk_profile; + st = stream_new_transcode(mfi, profile, hreq, offset, end_offset, stream_chunk_xcode_cb); if (!st) goto error; @@ -1191,23 +1196,18 @@ httpd_stream_file(struct httpd_request *hreq, int id) // Returns enum transcode_profile, but is just declared with int so we don't // need to include transcode.h in httpd_internal.h int -httpd_xcode_profile_get(const char *user_agent, const char *address, const char *accept_codecs, const char *codec) +httpd_xcode_profile_get(struct httpd_request *hreq) { - enum transcode_profile profile; struct player_speaker_info spk; int ret; - profile = transcode_needed(user_agent, accept_codecs, codec); - if (profile == XCODE_NONE) - return profile; - - DPRINTF(E_DBG, L_HTTPD, "Checking if client '%s' is a speaker\n", address); + DPRINTF(E_DBG, L_HTTPD, "Checking if client '%s' is a speaker\n", hreq->peer_address); // A Roku Soundbridge may also be RCP device/speaker for which the user may // have set a prefered streaming format - ret = player_speaker_get_byaddress(&spk, address); + ret = player_speaker_get_byaddress(&spk, hreq->peer_address); if (ret < 0) - return profile; + return XCODE_NONE; if (spk.format == MEDIA_FORMAT_WAV) return XCODE_WAV; @@ -1216,7 +1216,7 @@ httpd_xcode_profile_get(const char *user_agent, const char *address, const char if (spk.format == MEDIA_FORMAT_ALAC) return XCODE_MP4_ALAC; - return profile; + return XCODE_NONE; } struct evbuffer * diff --git a/src/httpd_daap.c b/src/httpd_daap.c index 5924e9e6..d5eee491 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -1149,6 +1149,7 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) const char *accept_codecs; const char *tag; size_t len; + enum transcode_profile spk_profile; enum transcode_profile profile; struct transcode_metadata_string xcode_metadata; struct media_quality quality = { 0 }; @@ -1222,6 +1223,8 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) accept_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); } + spk_profile = httpd_xcode_profile_get(hreq); + nsongs = 0; while ((ret = db_query_fetch_file(&dbmfi, &qp)) == 0) { @@ -1229,13 +1232,16 @@ daap_reply_songlist_generic(struct httpd_request *hreq, int playlist) // Not sure if the is_remote path is really needed. Note that if you // change the below you might need to do the same in rsp_reply_playlist() - profile = s->is_remote ? XCODE_WAV : httpd_xcode_profile_get(hreq->user_agent, hreq->peer_address, accept_codecs, dbmfi.codectype); + profile = s->is_remote ? XCODE_WAV : transcode_needed(hreq->user_agent, accept_codecs, dbmfi.codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_DAAP, "Cannot transcode '%s', codec type is unknown\n", dbmfi.fname); } else if (profile != XCODE_NONE) { + if (spk_profile != XCODE_NONE) + profile = spk_profile; + if (safe_atou32(dbmfi.song_length, &len_ms) < 0) len_ms = 3 * 60 * 1000; // just a fallback default diff --git a/src/httpd_internal.h b/src/httpd_internal.h index b9a0ad93..fc10eeb1 100644 --- a/src/httpd_internal.h +++ b/src/httpd_internal.h @@ -207,7 +207,7 @@ void httpd_stream_file(struct httpd_request *hreq, int id); int -httpd_xcode_profile_get(const char *user_agent, const char *address, const char *accept_codecs, const char *codec); +httpd_xcode_profile_get(struct httpd_request *hreq); void httpd_request_handler_set(struct httpd_request *hreq); diff --git a/src/httpd_rsp.c b/src/httpd_rsp.c index 1a78ab75..cd8c27bf 100644 --- a/src/httpd_rsp.c +++ b/src/httpd_rsp.c @@ -415,7 +415,7 @@ rsp_reply_db(struct httpd_request *hreq) } static int -item_add(xml_node *parent, struct query_params *qp, const char *user_agent, const char *address, const char *accept_codecs, int mode) +item_add(xml_node *parent, struct query_params *qp, enum transcode_profile spk_profile, const char *user_agent, const char *accept_codecs, int mode) { struct media_quality quality = { 0 }; struct db_media_file_info dbmfi; @@ -432,13 +432,16 @@ item_add(xml_node *parent, struct query_params *qp, const char *user_agent, cons if (ret != 0) return ret; - profile = httpd_xcode_profile_get(user_agent, address, accept_codecs, dbmfi.codectype); + profile = transcode_needed(user_agent, accept_codecs, dbmfi.codectype); if (profile == XCODE_UNKNOWN) { DPRINTF(E_LOG, L_DAAP, "Cannot transcode '%s', codec type is unknown\n", dbmfi.fname); } else if (profile != XCODE_NONE) { + if (spk_profile != XCODE_NONE) + profile = spk_profile; // User has configured a specific transcode format for this speaker + orgcodec = dbmfi.codectype; if (safe_atou32(dbmfi.song_length, &len_ms) < 0) @@ -485,6 +488,7 @@ rsp_reply_playlist(struct httpd_request *hreq) struct query_params qp; const char *param; const char *accept_codecs; + enum transcode_profile spk_profile; xml_node *response; xml_node *items; int mode; @@ -494,6 +498,7 @@ rsp_reply_playlist(struct httpd_request *hreq) memset(&qp, 0, sizeof(struct query_params)); accept_codecs = httpd_header_find(hreq->in_headers, "Accept-Codecs"); + spk_profile = httpd_xcode_profile_get(hreq); ret = safe_atoi32(hreq->path_parts[2], &qp.id); if (ret < 0) @@ -555,7 +560,7 @@ rsp_reply_playlist(struct httpd_request *hreq) items = xml_new_node(response, "items", NULL); do { - ret = item_add(items, &qp, hreq->user_agent, hreq->peer_address, accept_codecs, mode); + ret = item_add(items, &qp, spk_profile, hreq->user_agent, accept_codecs, mode); } while (ret == 0); From 7dd34792ea0d334ec436b4e32e3de257a8b4c3a4 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Thu, 18 Jan 2024 23:14:58 +0100 Subject: [PATCH 12/12] [cache] Multitreaded header encoding --- src/cache.c | 276 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 169 insertions(+), 107 deletions(-) diff --git a/src/cache.c b/src/cache.c index 8b45fc31..8290a122 100644 --- a/src/cache.c +++ b/src/cache.c @@ -93,6 +93,19 @@ struct cache_artwork_stash uint8_t *data; }; +struct cache_xcode_job +{ + const char *format; + char *file_path; + int file_id; + + struct event *ev; + bool is_encoding; + + struct evbuffer *header; +}; + + /* --------------------------------- GLOBALS -------------------------------- */ // cache thread @@ -188,12 +201,13 @@ static struct cache_db_def cache_artwork_db_def[] = { // Transcoding cache #define CACHE_XCODE_VERSION 1 +#define CACHE_XCODE_NTHREADS 4 +#define CACHE_XCODE_FORMAT_MP4 "mp4" static sqlite3 *cache_xcode_hdl; static struct event *cache_xcode_updateev; static struct event *cache_xcode_prepareev; +static struct cache_xcode_job cache_xcode_jobs[CACHE_XCODE_NTHREADS]; static bool cache_xcode_is_enabled; -static bool cache_xcode_prepare_is_running; -static int cache_xcode_last_file; static struct cache_db_def cache_xcode_db_def[] = { DB_DEF_ADMIN, { @@ -212,7 +226,8 @@ static struct cache_db_def cache_xcode_db_def[] = { " timestamp INTEGER DEFAULT 0," " file_id INTEGER DEFAULT 0," " format VARCHAR(255) NOT NULL," - " header BLOB" + " header BLOB," + " UNIQUE(file_id, format) ON CONFLICT REPLACE" ");", "DROP TABLE IF EXISTS data;", }, @@ -856,6 +871,24 @@ cache_daap_update_cb(int fd, short what, void *arg) DPRINTF(E_INFO, L_CACHE, "DAAP cache updated\n"); } + +/* ----------------------- Caching of transcoded data ----------------------- */ + +static void +xcode_job_clear(struct cache_xcode_job *job) +{ + free(job->file_path); + if (job->header) + evbuffer_free(job->header); + + // Can't just memset to zero, because *ev is persistent + job->format = NULL; + job->file_path = NULL; + job->file_id = 0; + job->header = NULL; + job->is_encoding = false; +} + static enum command_state xcode_header_get(void *arg, int *retval) { @@ -934,7 +967,7 @@ xcode_add_entry(sqlite3 *hdl, uint32_t id, uint32_t ts, const char *path) char *errmsg; int ret; - DPRINTF(E_LOG, L_CACHE, "Adding xcode file id %d, path '%s'\n", id, path); + DPRINTF(E_SPAM, L_CACHE, "Adding xcode file id %d, path '%s'\n", id, path); query = sqlite3_mprintf(Q_TMPL, id, ts, path); @@ -960,7 +993,7 @@ xcode_del_entry(sqlite3 *hdl, uint32_t id) char *errmsg; int ret; - DPRINTF(E_LOG, L_CACHE, "Deleting xcode file id %d\n", id); + DPRINTF(E_SPAM, L_CACHE, "Deleting xcode file id %d\n", id); sqlite3_snprintf(sizeof(query), query, Q_TMPL_FILES, (int)id); ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); @@ -1006,8 +1039,11 @@ xcode_sync_with_files(sqlite3 *hdl) size_t cachelist_len = 0; struct query_params qp = { .type = Q_ITEMS, .filter = "f.data_kind = 0", .order = "f.id" }; struct db_media_file_info dbmfi; + struct db_media_file_info *rowA; + struct cachelist *rowB; uint32_t id; uint32_t ts; + int cmp; int i; int ret; @@ -1033,42 +1069,47 @@ xcode_sync_with_files(sqlite3 *hdl) if (ret < 0) goto error; - // Loop while either list has remaining items - i = 0; - while (1) + // Loop while either list ("A" files list, "B" cache list) has remaining items + for(i = 0, cmp = 0;;) { - ret = db_query_fetch_file(&dbmfi, &qp); - if (ret != 0) // At end of files table (or error occured) - { - for (; i < cachelist_len; i++) - xcode_del_entry(hdl, cachelist[i].id); + if (cmp <= 0) + rowA = (db_query_fetch_file(&dbmfi, &qp) == 0) ? &dbmfi : NULL;; + if (cmp >= 0) + rowB = (i < cachelist_len) ? &cachelist[i++] : NULL; + if (!rowA && !rowB) + break; // Done with both lists - break; +#if 0 + if (rowA) + DPRINTF(E_DBG, L_CACHE, "cmp %d, rowA->id %s\n", cmp, rowA->id); + if (rowB) + DPRINTF(E_DBG, L_CACHE, "cmp %d, rowB->id %u, i %d, cachelist_len %zu\n", cmp, rowB->id, i, cachelist_len); +#endif + + if (rowA) + { + safe_atou32(rowA->id, &id); + safe_atou32(rowA->time_modified, &ts); } - safe_atou32(dbmfi.id, &id); - safe_atou32(dbmfi.time_modified, &ts); - - if (i == cachelist_len || cachelist[i].id > id) // At end of cache table or new file + cmp = 0; // In both lists - unless: + if (!rowB || (rowA && rowB->id > id)) // A had an item not in B { - xcode_add_entry(hdl, id, ts, dbmfi.path); + xcode_add_entry(hdl, id, ts, rowA->path); + cmp = -1; } - else if (cachelist[i].id < id) // Removed file + else if (!rowA || (rowB && rowB->id < id)) // B had an item not in A { - xcode_del_entry(hdl, cachelist[i].id); - i++; + xcode_del_entry(hdl, rowB->id); + cmp = 1; } - else if (cachelist[i].id == id && cachelist[i].ts < ts) // Modified file + else if (rowB->id == id && rowB->ts < ts) // Item in B is too old { - xcode_del_entry(hdl, cachelist[i].id); - xcode_add_entry(hdl, id, ts, dbmfi.path); - i++; - } - else // Found in both tables and timestamp in cache table is adequate - { - i++; + xcode_del_entry(hdl, rowB->id); + xcode_add_entry(hdl, id, ts, rowA->path); } } + db_query_end(&qp); free(cachelist); @@ -1081,135 +1122,157 @@ xcode_sync_with_files(sqlite3 *hdl) } static int -xcode_prepare_header(sqlite3 *hdl, const char *format, int id, const char *path) +xcode_header_save(sqlite3 *hdl, int file_id, const char *format, uint8_t *data, size_t datalen) { #define Q_TMPL "INSERT INTO data (timestamp, file_id, format, header) VALUES (?, ?, ?, ?);" - struct evbuffer *header = NULL; - sqlite3_stmt *stmt = NULL; - unsigned char *data = NULL; - size_t datalen = 0; + sqlite3_stmt *stmt; int ret; - DPRINTF(E_DBG, L_CACHE, "Preparing %s header for '%s' (file id %d)\n", format, path, id); - -#if 1 - if (strcmp(format, "mp4") == 0) - ret = transcode_prepare_header(&header, XCODE_MP4_ALAC, path); - else - ret = -1; - - // Proceed even if error, we also cache that - if (ret == 0) - { - datalen = evbuffer_get_length(header); - data = evbuffer_pullup(header, -1); - } -#elif - data = (unsigned char*)"dummy"; - datalen = 6; -#endif - ret = sqlite3_prepare_v2(hdl, Q_TMPL, -1, &stmt, 0); if (ret != SQLITE_OK) { - DPRINTF(E_LOG, L_CACHE, "Error preparing xcode_data for cache update: %s\n", sqlite3_errmsg(hdl)); - goto error; + DPRINTF(E_LOG, L_CACHE, "Error preparing xcode data for cache update: %s\n", sqlite3_errmsg(hdl)); + return -1; } sqlite3_bind_int(stmt, 1, (uint64_t)time(NULL)); - sqlite3_bind_int(stmt, 2, id); + sqlite3_bind_int(stmt, 2, file_id); sqlite3_bind_text(stmt, 3, format, -1, SQLITE_STATIC); sqlite3_bind_blob(stmt, 4, data, datalen, SQLITE_STATIC); ret = sqlite3_step(stmt); if (ret != SQLITE_DONE) { - DPRINTF(E_LOG, L_CACHE, "Error stepping xcode_data for cache update: %s\n", sqlite3_errmsg(hdl)); - goto error; + DPRINTF(E_LOG, L_CACHE, "Error stepping xcode data for cache update: %s\n", sqlite3_errmsg(hdl)); + return -1; } sqlite3_finalize(stmt); - if (header) - evbuffer_free(header); return 0; - - error: - if (stmt) - sqlite3_finalize(stmt); - if (header) - evbuffer_free(header); - return -1; #undef Q_TMPL } static int -xcode_prepare_next_header_impl(sqlite3 *hdl, const char *format) +xcode_file_next(int *file_id, char **file_path, sqlite3 *hdl, const char *format) { #define Q_TMPL "SELECT f.id, f.filepath, d.id FROM files f LEFT JOIN data d ON f.id = d.file_id AND d.format = '%q' WHERE d.id IS NULL LIMIT 1;" sqlite3_stmt *stmt; - char *query; - const char *file_path; - int file_id; + char query[256]; int ret; - query = sqlite3_mprintf(Q_TMPL, format); + sqlite3_snprintf(sizeof(query), query, Q_TMPL, format); ret = sqlite3_prepare_v2(hdl, query, -1, &stmt, 0); if (ret != SQLITE_OK) - goto error; + { + DPRINTF(E_LOG, L_CACHE, "Error occured while finding next file to prepare header for\n"); + return -1; + } ret = sqlite3_step(stmt); if (ret != SQLITE_ROW) { sqlite3_finalize(stmt); - sqlite3_free(query); return -1; // All done } - file_id = sqlite3_column_int(stmt, 0); - file_path = (const char *)sqlite3_column_text(stmt, 1); - - xcode_prepare_header(hdl, format, file_id, file_path); + *file_id = sqlite3_column_int(stmt, 0); + *file_path = strdup((char *)sqlite3_column_text(stmt, 1)); sqlite3_finalize(stmt); - sqlite3_free(query); - return file_id; - error: - DPRINTF(E_LOG, L_CACHE, "Error occured while preparing headers\n"); - sqlite3_free(query); - return -1; + // Save an empty header so next call to this function will return a new file + return xcode_header_save(hdl, *file_id, format, NULL, 0); #undef Q_TMPL } +// Thread: worker static void -xcode_prepare_next_header(void *arg) +xcode_worker(void *arg) { + struct cache_xcode_job *job = *(struct cache_xcode_job **)arg; int ret; - // Preparing headers can take very long, so we take one at a time, letting the - // event loop run in between - ret = xcode_prepare_next_header_impl(cache_xcode_hdl, "mp4"); - if (ret < 0 || ret == cache_xcode_last_file) + DPRINTF(E_DBG, L_CACHE, "Preparing %s header for '%s' (file id %d)\n", job->format, job->file_path, job->file_id); + + if (strcmp(job->format, CACHE_XCODE_FORMAT_MP4) == 0) { - DPRINTF(E_LOG, L_CACHE, "Header generation completed\n"); - cache_xcode_prepare_is_running = false; - return; + ret = transcode_prepare_header(&job->header, XCODE_MP4_ALAC, job->file_path); + if (ret < 0) + DPRINTF(E_LOG, L_CACHE, "Error preparing %s header for '%s' (file id %d)\n", job->format, job->file_path, job->file_id); } - // Used as failsafe to protect against infinite looping - cache_xcode_last_file = ret; + // Tell the cache thread that we are done. Only the cache thread can save the + // result to the DB. + event_active(job->ev, 0, 0); +} + +static void +cache_xcode_job_complete_cb(int fd, short what, void *arg) +{ + struct cache_xcode_job *job = arg; + uint8_t *data; + size_t datalen; + + if (job->header) + { +#if 1 + datalen = evbuffer_get_length(job->header); + data = evbuffer_pullup(job->header, -1); +#else + data = (unsigned char*)"dummy"; + datalen = 6; +#endif + xcode_header_save(cache_xcode_hdl, job->file_id, job->format, data, datalen); + } + + xcode_job_clear(job); // Makes the job available again + event_active(cache_xcode_prepareev, 0, 0); +} + +// Preparing headers can take very long, so we use worker threads. However, all +// DB access must be from the cache thread. So this function will find the next +// file from the db and then dispatch a thread for the encoding. +static void +cache_xcode_prepare_cb(int fd, short what, void *arg) +{ + struct cache_xcode_job *job = NULL; + bool is_encoding = false; + int ret; + int i; if (!cache_is_initialized) return; - event_active(cache_xcode_prepareev, 0, 0); -} + for (i = 0; i < ARRAY_SIZE(cache_xcode_jobs); i++) + { + if (cache_xcode_jobs[i].is_encoding) + is_encoding = true; + else if (!job) + job = &cache_xcode_jobs[i]; + } -static void -cache_xcode_prepare_cb(int fd, short what, void *arg) -{ - worker_execute(xcode_prepare_next_header, NULL, 0, 0); + if (!job) + return; // No available thread right now, wait for cache_xcode_job_complete_cb() + + ret = xcode_file_next(&job->file_id, &job->file_path, cache_xcode_hdl, CACHE_XCODE_FORMAT_MP4); + if (ret < 0) + { + if (!is_encoding) + DPRINTF(E_LOG, L_CACHE, "Header generation completed\n"); + + return; + } + else if (!is_encoding) + DPRINTF(E_LOG, L_CACHE, "Kicking off header generation\n"); + + job->is_encoding = true; + job->format = CACHE_XCODE_FORMAT_MP4; + + worker_execute(xcode_worker, &job, sizeof(struct cache_xcode_job *), 0); + + // Set off more threads + event_active(cache_xcode_prepareev, 0, 0); } static void @@ -1218,12 +1281,6 @@ cache_xcode_update_cb(int fd, short what, void *arg) if (xcode_sync_with_files(cache_xcode_hdl) < 0) return; - if (!cache_is_initialized || cache_xcode_prepare_is_running) - return; - - DPRINTF(E_LOG, L_CACHE, "Kicking off header generation\n"); - - cache_xcode_prepare_is_running = true; event_active(cache_xcode_prepareev, 0, 0); } @@ -1628,6 +1685,7 @@ static void * cache(void *arg) { int ret; + int i; ret = cache_open(); if (ret < 0) @@ -1651,6 +1709,8 @@ cache(void *arg) CHECK_NULL(L_CACHE, cache_xcode_updateev = evtimer_new(evbase_cache, cache_xcode_update_cb, NULL)); CHECK_NULL(L_CACHE, cache_xcode_prepareev = evtimer_new(evbase_cache, cache_xcode_prepare_cb, NULL)); CHECK_ERR(L_CACHE, event_priority_set(cache_xcode_prepareev, 0)); + for (i = 0; i < ARRAY_SIZE(cache_xcode_jobs); i++) + CHECK_NULL(L_CACHE, cache_xcode_jobs[i].ev = evtimer_new(evbase_cache, cache_xcode_job_complete_cb, &cache_xcode_jobs[i])); CHECK_ERR(L_CACHE, listener_add(cache_daap_listener_cb, LISTENER_DATABASE)); @@ -1666,6 +1726,8 @@ cache(void *arg) listener_remove(cache_daap_listener_cb); + for (i = 0; i < ARRAY_SIZE(cache_xcode_jobs); i++) + event_free(cache_xcode_jobs[i].ev); event_free(cache_xcode_prepareev); event_free(cache_xcode_updateev); event_free(cache_daap_updateev);