From 4472f9e9558bc3baf730d1fdd924cb7fac6552a6 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Mon, 15 Aug 2011 12:50:43 +0200 Subject: [PATCH 01/26] Fix infinite loop on unknown DACP property Missing strtok_r() call in the !dpm case. --- src/httpd_dacp.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/httpd_dacp.c b/src/httpd_dacp.c index 219c7a34..5486badb 100644 --- a/src/httpd_dacp.c +++ b/src/httpd_dacp.c @@ -1260,16 +1260,15 @@ dacp_reply_getproperty(struct evhttp_request *req, struct evbuffer *evbuf, char while (prop) { dpm = dacp_find_prop(prop, strlen(prop)); - if (!dpm) + if (dpm) { - DPRINTF(E_LOG, L_DACP, "Could not find requested property '%s'\n", prop); - continue; + if (dpm->propget) + dpm->propget(proplist, &status, mfi); + else + DPRINTF(E_WARN, L_DACP, "No getter method for DACP property %s\n", prop); } - - if (dpm->propget) - dpm->propget(proplist, &status, mfi); else - DPRINTF(E_WARN, L_DACP, "No getter method for DACP property %s\n", prop); + DPRINTF(E_LOG, L_DACP, "Could not find requested property '%s'\n", prop); prop = strtok_r(NULL, ",", &ptr); } From 29fe36522d8c4e567426a6cda4d16c7c631cf707 Mon Sep 17 00:00:00 2001 From: Anthony Doko Date: Sat, 27 Aug 2011 18:22:23 +0200 Subject: [PATCH 02/26] Add TV metadata patch for libav 0.7 --- ffmpeg/libav-0.7.patch | 81 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 ffmpeg/libav-0.7.patch diff --git a/ffmpeg/libav-0.7.patch b/ffmpeg/libav-0.7.patch new file mode 100644 index 00000000..2a54849b --- /dev/null +++ b/ffmpeg/libav-0.7.patch @@ -0,0 +1,81 @@ +--- libav/libavformat/mov.c 2011-08-09 14:03:47.622688230 -0700 ++++ libav/libavformat/mov.c 2011-08-09 14:09:33.181475099 -0700 +@@ -81,19 +81,46 @@ + + static const MOVParseTableEntry mov_default_parse_table[]; + +-static int mov_metadata_trkn(MOVContext *c, AVIOContext *pb, unsigned len) ++static int mov_metadata_trkn(MOVContext *c, AVIOContext *pb, unsigned len, const char *key) + { + char buf[16]; + + avio_rb16(pb); // unknown + snprintf(buf, sizeof(buf), "%d", avio_rb16(pb)); +- av_dict_set(&c->fc->metadata, "track", buf, 0); ++ av_dict_set(&c->fc->metadata, key, buf, 0); + + avio_rb16(pb); // total tracks + + return 0; + } + ++static int mov_metadata_int8(MOVContext *c, AVIOContext *pb, unsigned len, const char *key) ++{ ++ char buf[16]; ++ ++ /* bypass padding bytes */ ++ get_byte(pb); ++ get_byte(pb); ++ get_byte(pb); ++ ++ snprintf(buf, sizeof(buf-1), "%hu", get_byte(pb)); ++ buf[sizeof(buf)-1] = 0; ++ av_metadata_set2(&c->fc->metadata, key, buf, 0); ++ ++ return 0; ++} ++ ++static int mov_metadata_stik(MOVContext *c, AVIOContext *pb, unsigned len, const char *key) ++{ ++ char buf[16]; ++ ++ snprintf(buf, sizeof(buf-1), "%hu", get_byte(pb)); ++ buf[sizeof(buf)-1] = 0; ++ av_metadata_set2(&c->fc->metadata, key, buf, 0); ++ ++ return 0; ++} ++ + static const uint32_t mac_to_unicode[128] = { + 0x00C4,0x00C5,0x00C7,0x00C9,0x00D1,0x00D6,0x00DC,0x00E1, + 0x00E0,0x00E2,0x00E4,0x00E3,0x00E5,0x00E7,0x00E9,0x00E8, +@@ -140,7 +167,7 @@ + const char *key = NULL; + uint16_t str_size, langcode = 0; + uint32_t data_type = 0; +- int (*parse)(MOVContext*, AVIOContext*, unsigned) = NULL; ++ int (*parse)(MOVContext*, AVIOContext*, unsigned, const char *) = NULL; + + switch (atom.type) { + case MKTAG(0xa9,'n','a','m'): key = "title"; break; +@@ -162,6 +189,11 @@ + case MKTAG( 't','v','s','h'): key = "show"; break; + case MKTAG( 't','v','e','n'): key = "episode_id";break; + case MKTAG( 't','v','n','n'): key = "network"; break; ++ case MKTAG( 't','v','e','s'): key = "episode_sort"; ++ case MKTAG( 't','v','s','n'): key = "season_number"; ++ parse = mov_metadata_int8; break; ++ case MKTAG( 's','t','i','k'): key = "stik"; ++ parse = mov_metadata_stik; break; + case MKTAG( 't','r','k','n'): key = "track"; + parse = mov_metadata_trkn; break; + } +@@ -198,7 +230,7 @@ + str_size = FFMIN3(sizeof(str)-1, str_size, atom.size); + + if (parse) +- parse(c, pb, str_size); ++ parse(c, pb, str_size, key); + else { + if (data_type == 3 || (data_type == 0 && langcode < 0x800)) { // MAC Encoded + mov_read_mac_string(c, pb, str_size, str, sizeof(str)); From dbe22c2c0298585ee3f5fcbbccacfe9d18eba0f5 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 12:04:01 +0200 Subject: [PATCH 03/26] libav 0.7: use avformat_open_input() instead of av_open_input_file() --- src/artwork.c | 6 ++++++ src/filescanner_ffmpeg.c | 6 ++++++ src/transcode.c | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/src/artwork.c b/src/artwork.c index 8d25580d..c74c9b0f 100644 --- a/src/artwork.c +++ b/src/artwork.c @@ -500,7 +500,13 @@ artwork_get(char *filename, int max_w, int max_h, int format, struct evbuffer *e DPRINTF(E_DBG, L_ART, "Artwork request parameters: max w = %d, max h = %d\n", max_w, max_h); + src_ctx = NULL; + +#if LIBAVFORMAT_VERSION_MAJOR >= 53 || (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR >= 3) + ret = avformat_open_input(&src_ctx, filename, NULL, NULL); +#else ret = av_open_input_file(&src_ctx, filename, NULL, 0, NULL); +#endif if (ret < 0) { DPRINTF(E_WARN, L_ART, "Cannot open artwork file '%s': %s\n", filename, strerror(AVUNERROR(ret))); diff --git a/src/filescanner_ffmpeg.c b/src/filescanner_ffmpeg.c index d364c426..5a50b611 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/filescanner_ffmpeg.c @@ -311,7 +311,13 @@ scan_metadata_ffmpeg(char *file, struct media_file_info *mfi) int i; int ret; + ctx = NULL; + +#if LIBAVFORMAT_VERSION_MAJOR >= 53 || (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR >= 3) + ret = avformat_open_input(&ctx, file, NULL, NULL); +#else ret = av_open_input_file(&ctx, file, NULL, 0, NULL); +#endif if (ret != 0) { DPRINTF(E_WARN, L_SCAN, "Cannot open media file '%s': %s\n", file, strerror(AVUNERROR(ret))); diff --git a/src/transcode.c b/src/transcode.c index 80500a4a..2d704ed5 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -354,7 +354,11 @@ transcode_setup(struct media_file_info *mfi, off_t *est_size, int wavhdr) } memset(ctx, 0, sizeof(struct transcode_ctx)); +#if LIBAVFORMAT_VERSION_MAJOR >= 53 || (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR >= 3) + ret = avformat_open_input(&ctx->fmtctx, mfi->path, NULL, NULL); +#else ret = av_open_input_file(&ctx->fmtctx, mfi->path, NULL, 0, NULL); +#endif if (ret != 0) { DPRINTF(E_WARN, L_XCODE, "Could not open file %s: %s\n", mfi->fname, strerror(AVUNERROR(ret))); From bb8f9de8ec1a848f311e926bbd13fab094510e71 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 12:12:25 +0200 Subject: [PATCH 04/26] libav 0.7: use avformat_write_header() instead of av_write_header() --- src/artwork.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/artwork.c b/src/artwork.c index c74c9b0f..eefff340 100644 --- a/src/artwork.c +++ b/src/artwork.c @@ -265,6 +265,7 @@ artwork_rescale(AVFormatContext *src_ctx, int s, int out_w, int out_h, int forma dst->width = out_w; dst->height = out_h; +#if LIBAVFORMAT_VERSION_MAJOR <= 52 || (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR <= 1) ret = av_set_parameters(dst_ctx, NULL); if (ret < 0) { @@ -273,6 +274,7 @@ artwork_rescale(AVFormatContext *src_ctx, int s, int out_w, int out_h, int forma ret = -1; goto out_free_dst; } +#endif /* Open encoder */ ret = avcodec_open(dst, img_encoder); @@ -411,7 +413,11 @@ artwork_rescale(AVFormatContext *src_ctx, int s, int out_w, int out_h, int forma pkt.data = outbuf; pkt.size = ret; +#if LIBAVFORMAT_VERSION_MAJOR >= 53 || (LIBAVFORMAT_VERSION_MAJOR == 53 && LIBAVCODEC_VERSION_MINOR >= 3) + ret = avformat_write_header(dst_ctx, NULL); +#else ret = av_write_header(dst_ctx); +#endif if (ret != 0) { DPRINTF(E_LOG, L_ART, "Could not write artwork header: %s\n", strerror(AVUNERROR(ret))); From ed20d3f7de65dded6bcf322a812726f067ee73bd Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 12:20:15 +0200 Subject: [PATCH 05/26] libav 0.7: use av_get_bytes_per_sample() instead of av_get_bits_per_sample_fmt() --- src/filescanner_ffmpeg.c | 4 +++- src/transcode.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/filescanner_ffmpeg.c b/src/filescanner_ffmpeg.c index 5a50b611..08063e54 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/filescanner_ffmpeg.c @@ -410,7 +410,9 @@ scan_metadata_ffmpeg(char *file, struct media_file_info *mfi) mfi->samplerate = audio_stream->codec->sample_rate; /* Try sample format first */ -#if LIBAVCODEC_VERSION_MAJOR >= 53 +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 4) + mfi->bits_per_sample = 8 * av_get_bytes_per_sample(audio_stream->codec->sample_fmt); +#elif LIBAVCODEC_VERSION_MAJOR >= 53 mfi->bits_per_sample = av_get_bits_per_sample_fmt(audio_stream->codec->sample_fmt); #else mfi->bits_per_sample = av_get_bits_per_sample_format(audio_stream->codec->sample_fmt); diff --git a/src/transcode.c b/src/transcode.c index 2d704ed5..feb2ccbd 100644 --- a/src/transcode.c +++ b/src/transcode.c @@ -454,7 +454,9 @@ transcode_setup(struct media_file_info *mfi, off_t *est_size, int wavhdr) } ctx->need_resample = 1; -#if LIBAVCODEC_VERSION_MAJOR >= 53 +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 4) + ctx->input_size = ctx->acodec->channels * av_get_bytes_per_sample(ctx->acodec->sample_fmt); +#elif LIBAVCODEC_VERSION_MAJOR >= 53 ctx->input_size = ctx->acodec->channels * av_get_bits_per_sample_fmt(ctx->acodec->sample_fmt) / 8; #else ctx->input_size = ctx->acodec->channels * av_get_bits_per_sample_format(ctx->acodec->sample_fmt) / 8; From a2f409dd0a61c9b8f1cc027d7f0c02083e3a2a1e Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 12:25:52 +0200 Subject: [PATCH 06/26] libav 0.7: use av_dump_format() instead of dump_format() --- src/filescanner_ffmpeg.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/filescanner_ffmpeg.c b/src/filescanner_ffmpeg.c index 08063e54..feb2aacc 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/filescanner_ffmpeg.c @@ -336,7 +336,11 @@ scan_metadata_ffmpeg(char *file, struct media_file_info *mfi) #if 0 /* Dump input format as determined by ffmpeg */ +# if LIBAVFORMAT_VERSION_MAJOR >= 52 || (LIBAVFORMAT_VERSION_MAJOR == 52 && LIBAVCODEC_VERSION_MINOR >= 101) + av_dump_format(ctx, 0, file, 0); +# else dump_format(ctx, 0, file, FALSE); +# endif #endif DPRINTF(E_DBG, L_SCAN, "File has %d streams\n", ctx->nb_streams); From 0b2df545249c2809c00fca2a31536f046ee7f5c8 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 12:31:43 +0200 Subject: [PATCH 07/26] libav 0.7: switch to generic AVDictionary for metadata handling --- src/filescanner_ffmpeg.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/filescanner_ffmpeg.c b/src/filescanner_ffmpeg.c index feb2aacc..33aad4d6 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/filescanner_ffmpeg.c @@ -203,9 +203,17 @@ static const struct metadata_map md_map_id3[] = static int +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 5) +extract_metadata_core(struct media_file_info *mfi, AVDictionary *md, const struct metadata_map *md_map) +#else extract_metadata_core(struct media_file_info *mfi, AVMetadata *md, const struct metadata_map *md_map) +#endif { +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 5) + AVDictionaryEntry *mdt; +#else AVMetadataTag *mdt; +#endif char **strval; uint32_t *intval; int mdcount; @@ -215,7 +223,11 @@ extract_metadata_core(struct media_file_info *mfi, AVMetadata *md, const struct #if 0 /* Dump all the metadata reported by ffmpeg */ mdt = NULL; +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 5) + while ((mdt = av_dict_get(md, "", mdt, AV_DICT_IGNORE_SUFFIX)) != NULL) +#else while ((mdt = av_metadata_get(md, "", mdt, AV_METADATA_IGNORE_SUFFIX)) != NULL) +#endif fprintf(stderr, " -> %s = %s\n", mdt->key, mdt->value); #endif @@ -224,7 +236,11 @@ extract_metadata_core(struct media_file_info *mfi, AVMetadata *md, const struct /* Extract actual metadata */ for (i = 0; md_map[i].key != NULL; i++) { +#if LIBAVUTIL_VERSION_MAJOR >= 51 || (LIBAVUTIL_VERSION_MAJOR == 51 && LIBAVUTIL_VERSION_MINOR >= 5) + mdt = av_dict_get(md, md_map[i].key, NULL, 0); +#else mdt = av_metadata_get(md, md_map[i].key, NULL, 0); +#endif if (mdt == NULL) continue; From 3caae459be85bf358b4183979c8bb09a82c5fcdd Mon Sep 17 00:00:00 2001 From: Peter Carmichael Date: Sun, 5 Dec 2010 12:22:52 +0100 Subject: [PATCH 08/26] Add DB maintenance routines --- src/db.c | 29 +++++++++++++++++++++++++++++ src/db.h | 4 ++++ 2 files changed, 33 insertions(+) diff --git a/src/db.c b/src/db.c index 57a93dea..7bc81f54 100644 --- a/src/db.c +++ b/src/db.c @@ -568,6 +568,35 @@ db_exec(const char *query, char **errmsg) } +/* Maintenance and DB hygiene */ +static void +db_analyze(void) +{ + char *query = "ANALYZE;"; + char *errmsg; + int ret; + + DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); + + ret = db_exec(query, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "ANALYZE failed: %s\n", errmsg); + + sqlite3_free(errmsg); + } +} + +void +db_hook_post_scan(void) +{ + DPRINTF(E_DBG, L_DB, "Running post-scan DB maintenance tasks...\n"); + + db_analyze(); + + DPRINTF(E_DBG, L_DB, "Done with post-scan DB maintenance\n"); +} + void db_purge_cruft(time_t ref) { diff --git a/src/db.h b/src/db.h index b3468a3b..5e4bba89 100644 --- a/src/db.h +++ b/src/db.h @@ -286,6 +286,10 @@ unicode_fixup_mfi(struct media_file_info *mfi); void free_pli(struct playlist_info *pli, int content_only); +/* Maintenance and DB hygiene */ +void +db_hook_post_scan(void); + void db_purge_cruft(time_t ref); From 1909623d43f73da02391289969384ec376c94969 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 11 Jun 2011 16:22:05 +0200 Subject: [PATCH 09/26] Run ANALYZE after DB startup --- src/db.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db.c b/src/db.c index 7bc81f54..cef5bf44 100644 --- a/src/db.c +++ b/src/db.c @@ -4642,6 +4642,8 @@ db_init(void) } } + db_analyze(); + files = db_files_get_count(); pls = db_pl_get_count(); From 6e10252021b5a06a61c3fa2b3794cc7b6e59c111 Mon Sep 17 00:00:00 2001 From: Peter Carmichael Date: Sun, 5 Dec 2010 16:10:57 +0100 Subject: [PATCH 10/26] Perform post-bulk-scan DB maintenance --- src/filescanner.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/filescanner.c b/src/filescanner.c index c4c8dd6f..30ce89aa 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -811,6 +811,8 @@ filescanner(void *arg) bulk_scan(); + db_hook_post_scan(); + if (!scan_exit) { /* Enable inotify */ From 181b1e772f0126d2d652caca83ce93f2371ba520 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 11 Jun 2011 16:26:30 +0200 Subject: [PATCH 11/26] Add a logdomain for DB performance data --- src/logger.c | 2 +- src/logger.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/logger.c b/src/logger.c index 38aff3cb..1f040d68 100644 --- a/src/logger.c +++ b/src/logger.c @@ -43,7 +43,7 @@ static int threshold; static int console; static char *logfilename; static FILE *logfile; -static char *labels[] = { "config", "daap", "db", "httpd", "main", "mdns", "misc", "rsp", "scan", "xcode", "event", "remote", "dacp", "ffmpeg", "artwork", "player", "raop", "laudio", "dmap" }; +static char *labels[] = { "config", "daap", "db", "httpd", "main", "mdns", "misc", "rsp", "scan", "xcode", "event", "remote", "dacp", "ffmpeg", "artwork", "player", "raop", "laudio", "dmap", "dbperf" }; static int diff --git a/src/logger.h b/src/logger.h index 0c849c65..21ae92e3 100644 --- a/src/logger.h +++ b/src/logger.h @@ -25,8 +25,9 @@ #define L_RAOP 16 #define L_LAUDIO 17 #define L_DMAP 18 +#define L_DBPERF 19 -#define N_LOGDOMAINS 19 +#define N_LOGDOMAINS 20 /* Severities */ #define E_FATAL 0 From a2b7b811b839b89f2e24f747366d272683853f5a Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 11 Jun 2011 16:59:25 +0200 Subject: [PATCH 12/26] Add query plan to DB profiling callback --- src/db.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index cef5bf44..cce8c4f7 100644 --- a/src/db.c +++ b/src/db.c @@ -3675,8 +3675,57 @@ db_watch_enum_fetchwd(struct watch_enum *we, uint32_t *wd) static void db_xprofile(void *notused, const char *pquery, sqlite3_uint64 ptime) { - DPRINTF(E_DBG, L_DB, "SQL PROFILE query: %s\n", pquery); - DPRINTF(E_DBG, L_DB, "SQL PROFILE time: %" PRIu64 "\n", (uint64_t)ptime); + sqlite3_stmt *stmt; + char *query; + int ret; + + DPRINTF(E_DBG, L_DBPERF, "SQL PROFILE query: %s\n", pquery); + DPRINTF(E_DBG, L_DBPERF, "SQL PROFILE time: %" PRIu64 "\n", (uint64_t)ptime); + + if ((strncmp(pquery, "SELECT", 6) != 0) + && (strncmp(pquery, "UPDATE", 6) != 0) + && (strncmp(pquery, "DELETE", 6) != 0)) + return; + + /* Disable profiling callback */ + sqlite3_profile(hdl, NULL, NULL); + + query = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", pquery); + if (!query) + { + DPRINTF(E_DBG, L_DBPERF, "Query plan: Out of memory\n"); + + goto out; + } + + ret = db_blocking_prepare_v2(query, -1, &stmt, NULL); + sqlite3_free(query); + if (ret != SQLITE_OK) + { + DPRINTF(E_DBG, L_DBPERF, "Query plan: Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); + + goto out; + } + + DPRINTF(E_DBG, L_DBPERF, "Query plan:\n"); + + while ((ret = db_blocking_step(stmt)) == SQLITE_ROW) + { + DPRINTF(E_DBG, L_DBPERF, "(%d,%d,%d) %s\n", + sqlite3_column_int(stmt, 0), sqlite3_column_int(stmt, 1), sqlite3_column_int(stmt, 2), + sqlite3_column_text(stmt, 3)); + } + + if (ret != SQLITE_DONE) + DPRINTF(E_DBG, L_DBPERF, "Query plan: Could not step: %s\n", sqlite3_errmsg(hdl)); + + DPRINTF(E_DBG, L_DBPERF, "---\n"); + + sqlite3_finalize(stmt); + + out: + /* Reenable profiling callback */ + sqlite3_profile(hdl, db_xprofile, NULL); } #endif From d05634fddd7c0983bba14ae135a4b6800f3a4d1e Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 11 Jun 2011 17:25:24 +0200 Subject: [PATCH 13/26] Print query execution time in milliseconds --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index cce8c4f7..bc1c1be7 100644 --- a/src/db.c +++ b/src/db.c @@ -3680,7 +3680,7 @@ db_xprofile(void *notused, const char *pquery, sqlite3_uint64 ptime) int ret; DPRINTF(E_DBG, L_DBPERF, "SQL PROFILE query: %s\n", pquery); - DPRINTF(E_DBG, L_DBPERF, "SQL PROFILE time: %" PRIu64 "\n", (uint64_t)ptime); + DPRINTF(E_DBG, L_DBPERF, "SQL PROFILE time: %" PRIu64 " ms\n", ((uint64_t)ptime / 1000000)); if ((strncmp(pquery, "SELECT", 6) != 0) && (strncmp(pquery, "UPDATE", 6) != 0) From 333b7710bc3ead707005cbed388766297f71f4c8 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 11 Jun 2011 17:24:59 +0200 Subject: [PATCH 14/26] Queries must hit SQLITE_DONE to execute profiling callback --- src/db.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/db.c b/src/db.c index bc1c1be7..09915356 100644 --- a/src/db.c +++ b/src/db.c @@ -676,6 +676,11 @@ db_get_count(char *query) ret = sqlite3_column_int(stmt, 0); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); return ret; @@ -1638,6 +1643,11 @@ db_file_path_byid(int id) if (res) res = strdup(res); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); @@ -1679,6 +1689,11 @@ db_file_id_byquery(char *query) ret = sqlite3_column_int(stmt, 0); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); return ret; @@ -1823,6 +1838,11 @@ db_file_stamp_bypath(char *path) stamp = (time_t)sqlite3_column_int64(stmt, 0); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); @@ -1937,6 +1957,11 @@ db_file_fetch_byquery(char *query) } } +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); return mfi; @@ -2373,6 +2398,11 @@ db_pl_id_bypath(char *path, int *id) *id = sqlite3_column_int(stmt, 0); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); @@ -2950,6 +2980,11 @@ db_group_type_byid(int id) ret = sqlite3_column_int(stmt, 0); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); @@ -3073,6 +3108,11 @@ db_pairing_fetch_byguid(struct pairing_info *pi) pi->remote_id = strdup((char *)sqlite3_column_text(stmt, 0)); pi->name = strdup((char *)sqlite3_column_text(stmt, 1)); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); @@ -3161,6 +3201,11 @@ db_speaker_get(uint64_t id, int *selected, int *volume) *selected = sqlite3_column_int(stmt, 0); *volume = sqlite3_column_int(stmt, 1); +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); ret = 0; @@ -3451,6 +3496,11 @@ db_watch_get_bywd(struct watch_info *wi) } } +#ifdef DB_PROFILE + while (db_blocking_step(stmt) == SQLITE_ROW) + ; /* EMPTY */ +#endif + sqlite3_finalize(stmt); sqlite3_free(query); From 96c367f556625e0124e6a4729d7f704ab8ea81c6 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 12 Jun 2011 11:08:35 +0200 Subject: [PATCH 15/26] Kill useless database indexes --- src/db.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/db.c b/src/db.c index 09915356..b4b3ac20 100644 --- a/src/db.c +++ b/src/db.c @@ -3966,9 +3966,6 @@ db_perthread_deinit(void) " path VARCHAR(4096) NOT NULL" \ ");" -#define I_PATH \ - "CREATE INDEX IF NOT EXISTS idx_path ON files(path, idx);" - #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -3978,14 +3975,6 @@ db_perthread_deinit(void) #define I_PAIRING \ "CREATE INDEX IF NOT EXISTS idx_pairingguid ON pairings(guid);" -#define I_TITLESORT \ - "CREATE INDEX IF NOT EXISTS idx_titlesort ON files(title_sort);" - -#define I_ARTISTSORT \ - "CREATE INDEX IF NOT EXISTS idx_artistsort ON files(artist_sort);" - -#define I_ALBUMSORT \ - "CREATE INDEX IF NOT EXISTS idx_albumsort ON files(album_sort);" #define TRG_GROUPS_INSERT_FILES \ "CREATE TRIGGER update_groups_new_file AFTER INSERT ON files FOR EACH ROW" \ @@ -4043,13 +4032,9 @@ static const struct db_init_query db_init_queries[] = { T_SPEAKERS, "create table speakers" }, { T_INOTIFY, "create table inotify" }, - { I_PATH, "create file path index" }, { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, { I_PAIRING, "create pairing guid index" }, - { I_TITLESORT, "create file titlesort index" }, - { I_ARTISTSORT,"create file artistsort index" }, - { I_ALBUMSORT, "create file albumsort index" }, { TRG_GROUPS_INSERT_FILES, "create trigger update_groups_new_file" }, { TRG_GROUPS_UPDATE_FILES, "create trigger update_groups_update_file" }, From 1dfd27090e0572e8b69c0ce13a598647f3c9e9e3 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 12 Jun 2011 11:06:37 +0200 Subject: [PATCH 16/26] Speedup startup rescan --- src/db.c | 33 +++++++++++++++++++-------------- src/db.h | 6 +++--- src/filescanner.c | 5 +++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/db.c b/src/db.c index b4b3ac20..17082b85 100644 --- a/src/db.c +++ b/src/db.c @@ -1571,14 +1571,14 @@ db_file_inc_playcount(int id) } void -db_file_ping(char *path) +db_file_ping(int id) { -#define Q_TMPL "UPDATE files SET db_timestamp = %" PRIi64 ", disabled = 0 WHERE path = '%q';" +#define Q_TMPL "UPDATE files SET db_timestamp = %" PRIi64 ", disabled = 0 WHERE id = %d;" char *query; char *errmsg; int ret; - query = sqlite3_mprintf(Q_TMPL, (int64_t)time(NULL), path); + query = sqlite3_mprintf(Q_TMPL, (int64_t)time(NULL), id); if (!query) { DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); @@ -1590,7 +1590,7 @@ db_file_ping(char *path) ret = db_exec(query, &errmsg); if (ret != SQLITE_OK) - DPRINTF(E_LOG, L_DB, "Error pinging file '%s': %s\n", path, errmsg); + DPRINTF(E_LOG, L_DB, "Error pinging file ID %d: %s\n", id, errmsg); sqlite3_free(errmsg); sqlite3_free(query); @@ -1795,21 +1795,22 @@ db_file_id_byurl(char *url) #undef Q_TMPL } -time_t -db_file_stamp_bypath(char *path) +void +db_file_stamp_bypath(char *path, time_t *stamp, int *id) { -#define Q_TMPL "SELECT db_timestamp FROM files WHERE path = '%q';" +#define Q_TMPL "SELECT id, db_timestamp FROM files WHERE path = '%q';" char *query; sqlite3_stmt *stmt; - time_t stamp; int ret; + *stamp = 0; + query = sqlite3_mprintf(Q_TMPL, path); if (!query) { DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); - return 0; + return; } DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); @@ -1820,7 +1821,7 @@ db_file_stamp_bypath(char *path) DPRINTF(E_LOG, L_DB, "Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); sqlite3_free(query); - return 0; + return; } ret = db_blocking_step(stmt); @@ -1833,10 +1834,11 @@ db_file_stamp_bypath(char *path) sqlite3_finalize(stmt); sqlite3_free(query); - return 0; + return; } - stamp = (time_t)sqlite3_column_int64(stmt, 0); + *id = sqlite3_column_int(stmt, 0); + *stamp = (time_t)sqlite3_column_int64(stmt, 1); #ifdef DB_PROFILE while (db_blocking_step(stmt) == SQLITE_ROW) @@ -1846,8 +1848,6 @@ db_file_stamp_bypath(char *path) sqlite3_finalize(stmt); sqlite3_free(query); - return stamp; - #undef Q_TMPL } @@ -3966,6 +3966,9 @@ db_perthread_deinit(void) " path VARCHAR(4096) NOT NULL" \ ");" +#define I_RESCAN \ + "CREATE INDEX IF NOT EXISTS idx_rescan ON files(path, db_timestamp);" + #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -4032,6 +4035,8 @@ static const struct db_init_query db_init_queries[] = { T_SPEAKERS, "create table speakers" }, { T_INOTIFY, "create table inotify" }, + { I_RESCAN, "create rescan index" }, + { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, { I_PAIRING, "create pairing guid index" }, diff --git a/src/db.h b/src/db.h index 5e4bba89..b3db97ec 100644 --- a/src/db.h +++ b/src/db.h @@ -326,7 +326,7 @@ void db_file_inc_playcount(int id); void -db_file_ping(char *path); +db_file_ping(int id); char * db_file_path_byid(int id); @@ -343,8 +343,8 @@ db_file_id_byfile(char *filename); int db_file_id_byurl(char *url); -time_t -db_file_stamp_bypath(char *path); +void +db_file_stamp_bypath(char *path, time_t *stamp, int *id); struct media_file_info * db_file_fetch_byid(int id); diff --git a/src/filescanner.c b/src/filescanner.c index 30ce89aa..4c865a0a 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -301,13 +301,14 @@ process_media_file(char *file, time_t mtime, off_t size, int compilation) char *filename; char *ext; time_t stamp; + int id; int ret; - stamp = db_file_stamp_bypath(file); + db_file_stamp_bypath(file, &stamp, &id); if (stamp >= mtime) { - db_file_ping(file); + db_file_ping(id); return; } From becdfdb06229f8c3c3a280102c10868fe8b6b26d Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 12 Jun 2011 11:22:57 +0200 Subject: [PATCH 17/26] Add new indexes for general speedup An index on songalbumid helps for joins on songalbumid on the files table; a covering index on disabled+media_kind+songalbumid helps in numerous queries all over the place. --- src/db.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/db.c b/src/db.c index 17082b85..488cd660 100644 --- a/src/db.c +++ b/src/db.c @@ -3969,6 +3969,12 @@ db_perthread_deinit(void) #define I_RESCAN \ "CREATE INDEX IF NOT EXISTS idx_rescan ON files(path, db_timestamp);" +#define I_SONGALBUMID \ + "CREATE INDEX IF NOT EXISTS idx_sai ON files(songalbumid);" + +#define I_STATEMKINDSAI \ + "CREATE INDEX IF NOT EXISTS idx_state_mkind_sai ON files(disabled, media_kind, songalbumid);" + #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -4036,6 +4042,8 @@ static const struct db_init_query db_init_queries[] = { T_INOTIFY, "create table inotify" }, { I_RESCAN, "create rescan index" }, + { I_SONGALBUMID, "create songalbumid index" }, + { I_STATEMKINDSAI, "create state/mkind/sai index" }, { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, From 617c02606031246685a918488a42b601d211dd9b Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 12 Jun 2011 11:30:13 +0200 Subject: [PATCH 18/26] Add covering indexes for *{,_sort} fields These indexes help queries searching on these fields and will act as covering indexes for sort strings, too. --- src/db.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/db.c b/src/db.c index 488cd660..c34b200b 100644 --- a/src/db.c +++ b/src/db.c @@ -3975,6 +3975,21 @@ db_perthread_deinit(void) #define I_STATEMKINDSAI \ "CREATE INDEX IF NOT EXISTS idx_state_mkind_sai ON files(disabled, media_kind, songalbumid);" +#define I_ARTIST \ + "CREATE INDEX IF NOT EXISTS idx_artist ON files(artist, artist_sort);" + +#define I_ALBUMARTIST \ + "CREATE INDEX IF NOT EXISTS idx_albumartist ON files(album_artist, album_artist_sort);" + +#define I_COMPOSER \ + "CREATE INDEX IF NOT EXISTS idx_composer ON files(composer, composer_sort);" + +#define I_TITLE \ + "CREATE INDEX IF NOT EXISTS idx_title ON files(title, title_sort);" + +#define I_ALBUM \ + "CREATE INDEX IF NOT EXISTS idx_album ON files(album, album_sort);" + #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -4045,6 +4060,12 @@ static const struct db_init_query db_init_queries[] = { I_SONGALBUMID, "create songalbumid index" }, { I_STATEMKINDSAI, "create state/mkind/sai index" }, + { I_ARTIST, "create artist index" }, + { I_ALBUMARTIST, "create album_artist index" }, + { I_COMPOSER, "create composer index" }, + { I_TITLE, "create title index" }, + { I_ALBUM, "create album index" }, + { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, { I_PAIRING, "create pairing guid index" }, From cbc7108fcfc8541c9fe07b6800fd23c03d4377c6 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Mon, 28 Mar 2011 19:06:48 +0200 Subject: [PATCH 19/26] Use table aliases in all queries, update RSP & DAAP filters to match --- src/RSP2SQL.g | 3 + src/daap_query.gperf | 54 ++++++++-------- src/db.c | 148 +++++++++++++++++++++---------------------- 3 files changed, 104 insertions(+), 101 deletions(-) diff --git a/src/RSP2SQL.g b/src/RSP2SQL.g index 8796dba3..807e8e8e 100644 --- a/src/RSP2SQL.g +++ b/src/RSP2SQL.g @@ -205,6 +205,7 @@ strcrit returns [ pANTLR3_STRING result, int valid ] } $result = field->factory->newRaw(field->factory); + $result->append8($result, "f."); $result->appendS($result, field); $result->append8($result, op); $result->append8($result, "'"); @@ -289,6 +290,7 @@ intcrit returns [ pANTLR3_STRING result, int valid ] } $result = field->factory->newRaw(field->factory); + $result->append8($result, "f."); $result->appendS($result, field); $result->append8($result, op); $result->appendS($result, $i->getText($i)); @@ -362,6 +364,7 @@ datecrit returns [ pANTLR3_STRING result, int valid ] } $result = field->factory->newRaw(field->factory); + $result->append8($result, "f."); $result->appendS($result, field); $result->append8($result, op); $result->append8($result, buf); diff --git a/src/daap_query.gperf b/src/daap_query.gperf index 3e79412e..5a58753d 100644 --- a/src/daap_query.gperf +++ b/src/daap_query.gperf @@ -10,30 +10,30 @@ %omit-struct-type struct dmap_query_field_map; %% -"dmap.itemname", "title", 0 -"dmap.itemid", "id", 1 -"daap.songalbum", "album", 0 -"daap.songalbumid", "songalbumid", 1 -"daap.songartist", "artist", 0 -"daap.songalbumartist", "album_artist", 0 -"daap.songbitrate", "bitrate", 1 -"daap.songcomment", "comment", 0 -"daap.songcompilation", "compilation", 1 -"daap.songcomposer", "composer", 0 -"daap.songdatakind", "data_kind", 1 -"daap.songdataurl", "url", 0 -"daap.songdateadded", "time_added", 1 -"daap.songdatemodified", "time_modified", 1 -"daap.songdescription", "description", 0 -"daap.songdisccount", "total_discs", 1 -"daap.songdiscnumber", "disc", 1 -"daap.songformat", "type", 0 -"daap.songgenre", "genre", 0 -"daap.songsamplerate", "samplerate", 1 -"daap.songsize", "file_size", 1 -"daap.songstoptime", "song_length", 1 -"daap.songtime", "song_length", 1 -"daap.songtrackcount", "total_tracks", 1 -"daap.songtracknumber", "track", 1 -"daap.songyear", "year", 1 -"com.apple.itunes.mediakind", "media_kind", 1 +"dmap.itemname", "f.title", 0 +"dmap.itemid", "f.id", 1 +"daap.songalbum", "f.album", 0 +"daap.songalbumid", "f.songalbumid", 1 +"daap.songartist", "f.artist", 0 +"daap.songalbumartist", "f.album_artist", 0 +"daap.songbitrate", "f.bitrate", 1 +"daap.songcomment", "f.comment", 0 +"daap.songcompilation", "f.compilation", 1 +"daap.songcomposer", "f.composer", 0 +"daap.songdatakind", "f.data_kind", 1 +"daap.songdataurl", "f.url", 0 +"daap.songdateadded", "f.time_added", 1 +"daap.songdatemodified", "f.time_modified", 1 +"daap.songdescription", "f.description", 0 +"daap.songdisccount", "f.total_discs", 1 +"daap.songdiscnumber", "f.disc", 1 +"daap.songformat", "f.type", 0 +"daap.songgenre", "f.genre", 0 +"daap.songsamplerate", "f.samplerate", 1 +"daap.songsize", "f.file_size", 1 +"daap.songstoptime", "f.song_length", 1 +"daap.songtime", "f.song_length", 1 +"daap.songtrackcount", "f.total_tracks", 1 +"daap.songtracknumber", "f.track", 1 +"daap.songyear", "f.year", 1 +"com.apple.itunes.mediakind", "f.media_kind", 1 diff --git a/src/db.c b/src/db.c index c34b200b..e4a12b18 100644 --- a/src/db.c +++ b/src/db.c @@ -260,9 +260,9 @@ static const struct col_type_map wi_cols_map[] = static const char *sort_clause[] = { "", - "ORDER BY title_sort ASC", - "ORDER BY album_sort ASC, disc ASC, track ASC", - "ORDER BY artist_sort ASC", + "ORDER BY f.title_sort ASC", + "ORDER BY f.album_sort ASC, f.disc ASC, f.track ASC", + "ORDER BY f.artist_sort ASC", }; static char *db_path; @@ -606,7 +606,7 @@ db_purge_cruft(time_t ref) char *queries[3] = { NULL, NULL, NULL }; char *queries_tmpl[3] = { - "DELETE FROM playlistitems WHERE playlistid IN (SELECT id FROM playlists WHERE type <> 1 AND db_timestamp < %" PRIi64 ");", + "DELETE FROM playlistitems WHERE playlistid IN (SELECT id FROM playlists p WHERE p.type <> 1 AND p.db_timestamp < %" PRIi64 ");", "DELETE FROM playlists WHERE type <> 1 AND db_timestamp < %" PRIi64 ";", "DELETE FROM files WHERE db_timestamp < %" PRIi64 ";" }; @@ -737,9 +737,9 @@ db_build_query_items(struct query_params *qp, char **q) int ret; if (qp->filter) - count = sqlite3_mprintf("SELECT COUNT(*) FROM files WHERE disabled = 0 AND %s;", qp->filter); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f WHERE f.disabled = 0 AND %s;", qp->filter); else - count = sqlite3_mprintf("SELECT COUNT(*) FROM files WHERE disabled = 0;"); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f WHERE f.disabled = 0;"); if (!count) { @@ -762,13 +762,13 @@ db_build_query_items(struct query_params *qp, char **q) sort = sort_clause[qp->sort]; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT * FROM files WHERE disabled = 0 AND %s %s %s;", qp->filter, sort, idx); + query = sqlite3_mprintf("SELECT f.* FROM files f WHERE f.disabled = 0 AND %s %s %s;", qp->filter, sort, idx); else if (idx) - query = sqlite3_mprintf("SELECT * FROM files WHERE disabled = 0 %s %s;", sort, idx); + query = sqlite3_mprintf("SELECT f.* FROM files f WHERE f.disabled = 0 %s %s;", sort, idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT * FROM files WHERE disabled = 0 AND %s %s;", qp->filter, sort); + query = sqlite3_mprintf("SELECT f.* FROM files f WHERE f.disabled = 0 AND %s %s;", qp->filter, sort); else - query = sqlite3_mprintf("SELECT * FROM files WHERE disabled = 0 %s;", sort); + query = sqlite3_mprintf("SELECT f.* FROM files f WHERE f.disabled = 0 %s;", sort); if (!query) { @@ -788,7 +788,7 @@ db_build_query_pls(struct query_params *qp, char **q) char *idx; int ret; - qp->results = db_get_count("SELECT COUNT(*) FROM playlists WHERE disabled = 0;"); + qp->results = db_get_count("SELECT COUNT(*) FROM playlists p WHERE p.disabled = 0;"); if (qp->results < 0) return -1; @@ -798,13 +798,13 @@ db_build_query_pls(struct query_params *qp, char **q) return -1; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT * FROM playlists WHERE disabled = 0 AND %s %s;", qp->filter, idx); + query = sqlite3_mprintf("SELECT f.* FROM playlists f WHERE f.disabled = 0 AND %s %s;", qp->filter, idx); else if (idx) - query = sqlite3_mprintf("SELECT * FROM playlists WHERE disabled = 0 %s;", idx); + query = sqlite3_mprintf("SELECT f.* FROM playlists f WHERE f.disabled = 0 %s;", idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT * FROM playlists WHERE disabled = 0 AND %s;", qp->filter); + query = sqlite3_mprintf("SELECT f.* FROM playlists f WHERE f.disabled = 0 AND %s;", qp->filter); else - query = sqlite3_mprintf("SELECT * FROM playlists WHERE disabled = 0;"); + query = sqlite3_mprintf("SELECT f.* FROM playlists f WHERE f.disabled = 0;"); if (!query) { @@ -826,11 +826,11 @@ db_build_query_plitems_plain(struct query_params *qp, char **q) int ret; if (qp->filter) - count = sqlite3_mprintf("SELECT COUNT(*) FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0 AND %s;", qp->id, qp->filter); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0 AND %s;", qp->id, qp->filter); else - count = sqlite3_mprintf("SELECT COUNT(*) FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0;", qp->id); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0;", qp->id); if (!count) { @@ -851,20 +851,20 @@ db_build_query_plitems_plain(struct query_params *qp, char **q) return -1; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT files.* FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0 AND %s ORDER BY playlistitems.id ASC %s;", + query = sqlite3_mprintf("SELECT f.* FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0 AND %s ORDER BY pi.id ASC %s;", qp->id, qp->filter, idx); else if (idx) - query = sqlite3_mprintf("SELECT files.* FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0 ORDER BY playlistitems.id ASC %s;", + query = sqlite3_mprintf("SELECT f.* FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0 ORDER BY pi.id ASC %s;", qp->id, idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT files.* FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0 AND %s ORDER BY playlistitems.id ASC;", + query = sqlite3_mprintf("SELECT f.* FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0 AND %s ORDER BY pi.id ASC;", qp->id, qp->filter); else - query = sqlite3_mprintf("SELECT files.* FROM files JOIN playlistitems ON files.path = playlistitems.filepath" - " WHERE playlistitems.playlistid = %d AND files.disabled = 0 ORDER BY playlistitems.id ASC;", + query = sqlite3_mprintf("SELECT f.* FROM files f JOIN playlistitems pi ON f.path = pi.filepath" + " WHERE pi.playlistid = %d AND f.disabled = 0 ORDER BY pi.id ASC;", qp->id); if (!query) @@ -893,7 +893,7 @@ db_build_query_plitems_smart(struct query_params *qp, char *smartpl_query, char else filter = "1 = 1"; - count = sqlite3_mprintf("SELECT COUNT(*) FROM files WHERE disabled = 0 AND %s AND %s;", filter, smartpl_query); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f WHERE f.disabled = 0 AND %s AND %s;", filter, smartpl_query); if (!count) { DPRINTF(E_LOG, L_DB, "Out of memory for count query string\n"); @@ -917,7 +917,7 @@ db_build_query_plitems_smart(struct query_params *qp, char *smartpl_query, char sort = sort_clause[qp->sort]; - query = sqlite3_mprintf("SELECT * FROM files WHERE disabled = 0 AND %s AND %s %s %s;", smartpl_query, filter, sort, idx); + query = sqlite3_mprintf("SELECT f.* FROM files f WHERE f.disabled = 0 AND %s AND %s %s %s;", smartpl_query, filter, sort, idx); if (!query) { DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); @@ -973,7 +973,7 @@ db_build_query_groups(struct query_params *qp, char **q) char *idx; int ret; - qp->results = db_get_count("SELECT COUNT(DISTINCT songalbumid) FROM files WHERE disabled = 0;"); + qp->results = db_get_count("SELECT COUNT(DISTINCT f.songalbumid) FROM files f WHERE f.disabled = 0;"); if (qp->results < 0) return -1; @@ -983,13 +983,13 @@ db_build_query_groups(struct query_params *qp, char **q) return -1; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND disabled = 0 AND %s %s;", G_ALBUMS, qp->filter, idx); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 AND %s %s;", G_ALBUMS, qp->filter, idx); else if (idx) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND disabled = 0 %s;", G_ALBUMS, idx); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 %s;", G_ALBUMS, idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND disabled = 0 AND %s;", G_ALBUMS, qp->filter); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 AND %s;", G_ALBUMS, qp->filter); else - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND disabled = 0;", G_ALBUMS); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0;", G_ALBUMS); if (!query) { @@ -1014,8 +1014,8 @@ db_build_query_groupitems(struct query_params *qp, char **q) switch (gt) { case G_ALBUMS: - count = sqlite3_mprintf("SELECT COUNT(*) FROM files JOIN groups ON files.songalbumid = groups.persistentid" - " WHERE groups.id = %d AND files.disabled = 0;", qp->id); + count = sqlite3_mprintf("SELECT COUNT(*) FROM files f JOIN groups g ON f.songalbumid = g.persistentid" + " WHERE g.id = %d AND f.disabled = 0;", qp->id); break; default: @@ -1039,8 +1039,8 @@ db_build_query_groupitems(struct query_params *qp, char **q) switch (gt) { case G_ALBUMS: - query = sqlite3_mprintf("SELECT files.* FROM files JOIN groups ON files.songalbumid = groups.persistentid" - " WHERE groups.id = %d AND files.disabled = 0;", qp->id); + query = sqlite3_mprintf("SELECT f.* FROM files f JOIN groups g ON f.songalbumid = g.persistentid" + " WHERE g.id = %d AND f.disabled = 0;", qp->id); break; } @@ -1067,9 +1067,9 @@ db_build_query_group_dirs(struct query_params *qp, char **q) switch (gt) { case G_ALBUMS: - count = sqlite3_mprintf("SELECT COUNT(DISTINCT(SUBSTR(files.path, 1, LENGTH(files.path) - LENGTH(files.fname) - 1)))" - " FROM files JOIN groups ON files.songalbumid = groups.persistentid" - " WHERE groups.id = %d AND files.disabled = 0;", qp->id); + count = sqlite3_mprintf("SELECT COUNT(DISTINCT(SUBSTR(f.path, 1, LENGTH(f.path) - LENGTH(f.fname) - 1)))" + " FROM files f JOIN groups g ON f.songalbumid = g.persistentid" + " WHERE g.id = %d AND f.disabled = 0;", qp->id); break; default: @@ -1093,9 +1093,9 @@ db_build_query_group_dirs(struct query_params *qp, char **q) switch (gt) { case G_ALBUMS: - query = sqlite3_mprintf("SELECT DISTINCT(SUBSTR(files.path, 1, LENGTH(files.path) - LENGTH(files.fname) - 1))" - " FROM files JOIN groups ON files.songalbumid = groups.persistentid" - " WHERE groups.id = %d AND files.disabled = 0;", qp->id); + query = sqlite3_mprintf("SELECT DISTINCT(SUBSTR(f.path, 1, LENGTH(f.path) - LENGTH(f.fname) - 1))" + " FROM files f JOIN groups g ON f.songalbumid = g.persistentid" + " WHERE g.id = %d AND f.disabled = 0;", qp->id); break; } @@ -1119,10 +1119,10 @@ db_build_query_browse(struct query_params *qp, char *field, char **q) int ret; if (qp->filter) - count = sqlite3_mprintf("SELECT COUNT(DISTINCT %s) FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != '' AND %s;", + count = sqlite3_mprintf("SELECT COUNT(DISTINCT f.%s) FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != '' AND %s;", field, field, qp->filter); else - count = sqlite3_mprintf("SELECT COUNT(DISTINCT %s) FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != '';", + count = sqlite3_mprintf("SELECT COUNT(DISTINCT f.%s) FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != '';", field, field); if (!count) @@ -1144,16 +1144,16 @@ db_build_query_browse(struct query_params *qp, char *field, char **q) return -1; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT DISTINCT %s, %s FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != ''" + query = sqlite3_mprintf("SELECT DISTINCT f.%s, f.%s FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != ''" " AND %s %s;", field, field, field, qp->filter, idx); else if (idx) - query = sqlite3_mprintf("SELECT DISTINCT %s, %s FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != ''" + query = sqlite3_mprintf("SELECT DISTINCT f.%s, f.%s FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != ''" " %s;", field, field, field, idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT DISTINCT %s, %s FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != ''" + query = sqlite3_mprintf("SELECT DISTINCT f.%s, f.%s FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != ''" " AND %s;", field, field, field, qp->filter); else - query = sqlite3_mprintf("SELECT DISTINCT %s, %s FROM files WHERE data_kind = 0 AND disabled = 0 AND %s != ''", + query = sqlite3_mprintf("SELECT DISTINCT f.%s, f.%s FROM files f WHERE f.data_kind = 0 AND f.disabled = 0 AND f.%s != ''", field, field, field); if (!query) @@ -1521,7 +1521,7 @@ db_query_fetch_string_sort(struct query_params *qp, char **string, char **sortst int db_files_get_count(void) { - return db_get_count("SELECT COUNT(*) FROM files WHERE disabled = 0;"); + return db_get_count("SELECT COUNT(*) FROM files f WHERE f.disabled = 0;"); } void @@ -1601,7 +1601,7 @@ db_file_ping(int id) char * db_file_path_byid(int id) { -#define Q_TMPL "SELECT path FROM files WHERE id = %d;" +#define Q_TMPL "SELECT f.path FROM files f WHERE f.id = %d;" char *query; sqlite3_stmt *stmt; char *res; @@ -1702,7 +1702,7 @@ db_file_id_byquery(char *query) int db_file_id_bypath(char *path) { -#define Q_TMPL "SELECT id FROM files WHERE path = '%q';" +#define Q_TMPL "SELECT f.id FROM files f WHERE f.path = '%q';" char *query; int ret; @@ -1726,7 +1726,7 @@ db_file_id_bypath(char *path) int db_file_id_byfilebase(char *filename, char *base) { -#define Q_TMPL "SELECT id FROM files WHERE path LIKE '%q/%%/%q';" +#define Q_TMPL "SELECT f.id FROM files f WHERE f.path LIKE '%q/%%/%q';" char *query; int ret; @@ -1750,7 +1750,7 @@ db_file_id_byfilebase(char *filename, char *base) int db_file_id_byfile(char *filename) { -#define Q_TMPL "SELECT id FROM files WHERE fname = '%q';" +#define Q_TMPL "SELECT f.id FROM files f WHERE f.fname = '%q';" char *query; int ret; @@ -1774,7 +1774,7 @@ db_file_id_byfile(char *filename) int db_file_id_byurl(char *url) { -#define Q_TMPL "SELECT id FROM files WHERE url = '%q';" +#define Q_TMPL "SELECT f.id FROM files f WHERE f.url = '%q';" char *query; int ret; @@ -1798,7 +1798,7 @@ db_file_id_byurl(char *url) void db_file_stamp_bypath(char *path, time_t *stamp, int *id) { -#define Q_TMPL "SELECT id, db_timestamp FROM files WHERE path = '%q';" +#define Q_TMPL "SELECT f.id, f.db_timestamp FROM files f WHERE f.path = '%q';" char *query; sqlite3_stmt *stmt; int ret; @@ -1970,7 +1970,7 @@ db_file_fetch_byquery(char *query) struct media_file_info * db_file_fetch_byid(int id) { -#define Q_TMPL "SELECT * FROM files WHERE id = %d;" +#define Q_TMPL "SELECT f.* FROM files f WHERE f.id = %d;" struct media_file_info *mfi; char *query; @@ -2276,14 +2276,14 @@ db_file_enable_bycookie(uint32_t cookie, char *path) int db_pl_get_count(void) { - return db_get_count("SELECT COUNT(*) FROM playlists WHERE disabled = 0;"); + return db_get_count("SELECT COUNT(*) FROM playlists p WHERE p.disabled = 0;"); } static int db_pl_count_items(int id) { -#define Q_TMPL "SELECT COUNT(*) FROM playlistitems JOIN files" \ - " ON playlistitems.filepath = files.path WHERE files.disabled = 0 AND playlistitems.playlistid = %d;" +#define Q_TMPL "SELECT COUNT(*) FROM playlistitems pi JOIN files f" \ + " ON pi.filepath = f.path WHERE f.disabled = 0 AND pi.playlistid = %d;" char *query; int ret; @@ -2307,7 +2307,7 @@ db_pl_count_items(int id) static int db_smartpl_count_items(const char *smartpl_query) { -#define Q_TMPL "SELECT COUNT(*) FROM files WHERE disabled = 0 AND %s;" +#define Q_TMPL "SELECT COUNT(*) FROM files f WHERE f.disabled = 0 AND %s;" char *query; int ret; @@ -2359,7 +2359,7 @@ db_pl_ping(int id) static int db_pl_id_bypath(char *path, int *id) { -#define Q_TMPL "SELECT id FROM playlists WHERE path = '%q';" +#define Q_TMPL "SELECT p.id FROM playlists p WHERE p.path = '%q';" char *query; sqlite3_stmt *stmt; int ret; @@ -2537,7 +2537,7 @@ db_pl_fetch_byquery(char *query) struct playlist_info * db_pl_fetch_bypath(char *path) { -#define Q_TMPL "SELECT * FROM playlists WHERE path = '%q';" +#define Q_TMPL "SELECT p.* FROM playlists p WHERE p.path = '%q';" struct playlist_info *pli; char *query; @@ -2561,7 +2561,7 @@ db_pl_fetch_bypath(char *path) struct playlist_info * db_pl_fetch_byid(int id) { -#define Q_TMPL "SELECT * FROM playlists WHERE id = %d;" +#define Q_TMPL "SELECT p.* FROM playlists p WHERE p.id = %d;" struct playlist_info *pli; char *query; @@ -2585,7 +2585,7 @@ db_pl_fetch_byid(int id) struct playlist_info * db_pl_fetch_bytitlepath(char *title, char *path) { -#define Q_TMPL "SELECT * FROM playlists WHERE title = '%q' AND path = '%q';" +#define Q_TMPL "SELECT p.* FROM playlists p WHERE p.title = '%q' AND p.path = '%q';" struct playlist_info *pli; char *query; @@ -2609,7 +2609,7 @@ db_pl_fetch_bytitlepath(char *title, char *path) int db_pl_add(char *title, char *path, int *id) { -#define QDUP_TMPL "SELECT COUNT(*) FROM playlists WHERE title = '%q' AND path = '%q';" +#define QDUP_TMPL "SELECT COUNT(*) FROM playlists p WHERE p.title = '%q' AND p.path = '%q';" #define QADD_TMPL "INSERT INTO playlists (title, type, query, db_timestamp, disabled, path, idx, special_id)" \ " VALUES ('%q', 0, NULL, %" PRIi64 ", 0, '%q', 0, 0);" char *query; @@ -2708,7 +2708,7 @@ db_pl_add_item_bypath(int plid, char *path) int db_pl_add_item_byid(int plid, int fileid) { -#define Q_TMPL "INSERT INTO playlistitems (playlistid, filepath) VALUES (%d, (SELECT path FROM files WHERE id = %d));" +#define Q_TMPL "INSERT INTO playlistitems (playlistid, filepath) VALUES (%d, (SELECT f.path FROM files f WHERE f.id = %d));" char *query; char *errmsg; int ret; @@ -2941,7 +2941,7 @@ db_groups_clear(void) enum group_type db_group_type_byid(int id) { -#define Q_TMPL "SELECT type FROM groups WHERE id = '%d';" +#define Q_TMPL "SELECT g.type FROM groups g WHERE g.id = '%d';" char *query; sqlite3_stmt *stmt; int ret; @@ -3071,7 +3071,7 @@ db_pairing_add(struct pairing_info *pi) int db_pairing_fetch_byguid(struct pairing_info *pi) { -#define Q_TMPL "SELECT * FROM pairings WHERE guid = '%q';" +#define Q_TMPL "SELECT p.* FROM pairings p WHERE p.guid = '%q';" char *query; sqlite3_stmt *stmt; int ret; @@ -3162,7 +3162,7 @@ db_speaker_save(uint64_t id, int selected, int volume) int db_speaker_get(uint64_t id, int *selected, int *volume) { -#define Q_TMPL "SELECT selected, volume FROM speakers WHERE id = %" PRIi64 ";" +#define Q_TMPL "SELECT s.selected, s.volume FROM speakers s WHERE s.id = %" PRIi64 ";" sqlite3_stmt *stmt; char *query; int ret; @@ -4018,15 +4018,15 @@ db_perthread_deinit(void) #define Q_PL2 \ "INSERT INTO playlists (id, title, type, query, db_timestamp, path, idx, special_id)" \ - " VALUES(2, 'Music', 1, 'media_kind = 1', 0, '', 0, 6);" + " VALUES(2, 'Music', 1, 'f.media_kind = 1', 0, '', 0, 6);" #define Q_PL3 \ "INSERT INTO playlists (id, title, type, query, db_timestamp, path, idx, special_id)" \ - " VALUES(3, 'Movies', 1, 'media_kind = 2', 0, '', 0, 4);" + " VALUES(3, 'Movies', 1, 'f.media_kind = 2', 0, '', 0, 4);" #define Q_PL4 \ "INSERT INTO playlists (id, title, type, query, db_timestamp, path, idx, special_id)" \ - " VALUES(4, 'TV Shows', 1, 'media_kind = 64', 0, '', 0, 5);" + " VALUES(4, 'TV Shows', 1, 'f.media_kind = 64', 0, '', 0, 5);" /* These are the remaining automatically-created iTunes playlists, but * their query is unknown From 664067fd888dc2c2dfe5f663705500d41dab1171 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 26 Jun 2011 16:22:56 +0200 Subject: [PATCH 20/26] Rewrite group queries, remove JOIN, add index SQLite has trouble optimizing the query when written with JOIN, but does pretty well using the indexes when written without JOIN. Add an index for the query, too. --- src/db.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index e4a12b18..f9a4d1dd 100644 --- a/src/db.c +++ b/src/db.c @@ -983,13 +983,13 @@ db_build_query_groups(struct query_params *qp, char **q) return -1; if (idx && qp->filter) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 AND %s %s;", G_ALBUMS, qp->filter, idx); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f, groups g WHERE f.songalbumid = g.persistentid AND g.type = %d AND f.disabled = 0 AND %s GROUP BY f.album, g.name %s;", G_ALBUMS, qp->filter, idx); else if (idx) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 %s;", G_ALBUMS, idx); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f, groups g WHERE f.songalbumid = g.persistentid AND g.type = %d AND f.disabled = 0 GROUP BY f.album, g.name %s;", G_ALBUMS, idx); else if (qp->filter) - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0 AND %s;", G_ALBUMS, qp->filter); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f, groups g WHERE f.songalbumid = g.persistentid AND g.type = %d AND f.disabled = 0 AND %s GROUP BY f.album, g.name;", G_ALBUMS, qp->filter); else - query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f JOIN groups g ON f.songalbumid = g.persistentid GROUP BY f.album, g.name HAVING g.type = %d AND f.disabled = 0;", G_ALBUMS); + query = sqlite3_mprintf("SELECT COUNT(*), g.id, g.persistentid, f.album_artist, g.name FROM files f, groups g WHERE f.songalbumid = g.persistentid AND g.type = %d AND f.disabled = 0 GROUP BY f.album, g.name;", G_ALBUMS); if (!query) { @@ -3996,6 +3996,9 @@ db_perthread_deinit(void) #define I_PLITEMID \ "CREATE INDEX IF NOT EXISTS idx_playlistid ON playlistitems(playlistid, filepath);" +#define I_GRP_TYPE_PERSIST \ + "CREATE INDEX IF NOT EXISTS idx_grp_type_persist ON groups(type, persistentid);" + #define I_PAIRING \ "CREATE INDEX IF NOT EXISTS idx_pairingguid ON pairings(guid);" @@ -4068,6 +4071,9 @@ static const struct db_init_query db_init_queries[] = { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, + + { I_GRP_TYPE_PERSIST, "create groups type/persistentid index" }, + { I_PAIRING, "create pairing guid index" }, { TRG_GROUPS_INSERT_FILES, "create trigger update_groups_new_file" }, From efcd463946edf739420b9708658db552fa2bf735 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 26 Jun 2011 16:20:31 +0200 Subject: [PATCH 21/26] Add indexes for playlists --- src/db.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/db.c b/src/db.c index f9a4d1dd..6517188d 100644 --- a/src/db.c +++ b/src/db.c @@ -3990,6 +3990,12 @@ db_perthread_deinit(void) #define I_ALBUM \ "CREATE INDEX IF NOT EXISTS idx_album ON files(album, album_sort);" +#define I_PL_PATH \ + "CREATE INDEX IF NOT EXISTS idx_pl_path ON playlists(path);" + +#define I_PL_DISABLED \ + "CREATE INDEX IF NOT EXISTS idx_pl_disabled ON playlists(disabled);" + #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -4069,6 +4075,9 @@ static const struct db_init_query db_init_queries[] = { I_TITLE, "create title index" }, { I_ALBUM, "create album index" }, + { I_PL_PATH, "create playlist path index" }, + { I_PL_DISABLED, "create playlist state index" }, + { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, From ad4e15c3623effb153580e6ed013620856904bcc Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Fri, 15 Jul 2011 18:11:29 +0200 Subject: [PATCH 22/26] Handle database upgrade v12 -> v13 --- src/db.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index 6517188d..fc7b3934 100644 --- a/src/db.c +++ b/src/db.c @@ -4045,9 +4045,9 @@ db_perthread_deinit(void) " VALUES(8, 'Purchased', 0, 'media_kind = 1024', 0, '', 0, 8);" */ -#define SCHEMA_VERSION 12 +#define SCHEMA_VERSION 13 #define Q_SCVER \ - "INSERT INTO admin (key, value) VALUES ('schema_version', '12');" + "INSERT INTO admin (key, value) VALUES ('schema_version', '13');" struct db_init_query { char *query; @@ -4623,6 +4623,92 @@ db_upgrade_v12(void) #undef Q_DUMP } +/* Upgrade from schema v12 to v13 */ + +#define U_V13_DROP_IDX_PATH \ + "DROP INDEX idx_path;" + +#define U_V13_DROP_IDX_TS \ + "DROP INDEX idx_titlesort;" + +#define U_V13_DROP_IDX_AS \ + "DROP INDEX idx_artistsort;" + +#define U_V13_DROP_IDX_BS \ + "DROP INDEX idx_albumsort;" + +#define U_V13_IDX_RESCAN \ + "CREATE INDEX IF NOT EXISTS idx_rescan ON files(path, db_timestamp);" + +#define U_V13_IDX_SONGALBUMID \ + "CREATE INDEX IF NOT EXISTS idx_sai ON files(songalbumid);" + +#define U_V13_IDX_STATEMKINDSAI \ + "CREATE INDEX IF NOT EXISTS idx_state_mkind_sai ON files(disabled, media_kind, songalbumid);" + +#define U_V13_IDX_ARTIST \ + "CREATE INDEX IF NOT EXISTS idx_artist ON files(artist, artist_sort);" + +#define U_V13_IDX_ALBUMARTIST \ + "CREATE INDEX IF NOT EXISTS idx_albumartist ON files(album_artist, album_artist_sort);" + +#define U_V13_IDX_COMPOSER \ + "CREATE INDEX IF NOT EXISTS idx_composer ON files(composer, composer_sort);" + +#define U_V13_IDX_TITLE \ + "CREATE INDEX IF NOT EXISTS idx_title ON files(title, title_sort);" + +#define U_V13_IDX_ALBUM \ + "CREATE INDEX IF NOT EXISTS idx_album ON files(album, album_sort);" + +#define U_V13_IDX_GRP_TYPE_PERSIST \ + "CREATE INDEX IF NOT EXISTS idx_grp_type_persist ON groups(type, persistentid);" + +#define U_V13_IDX_PL_PATH \ + "CREATE INDEX IF NOT EXISTS idx_pl_path ON playlists(path);" + +#define U_V13_IDX_PL_DISABLED \ + "CREATE INDEX IF NOT EXISTS idx_pl_disabled ON playlists(disabled);" + +#define U_V13_PL2 \ + "UPDATE playlists SET query = 'f.media_kind = 1' where id = 2;" + +#define U_V13_PL3 \ + "UPDATE playlists SET query = 'f.media_kind = 2' where id = 3;" + +#define U_V13_PL4 \ + "UPDATE playlists SET query = 'f.media_kind = 64' where id = 4;" + +#define U_V13_SCVER \ + "UPDATE admin SET value = '13' WHERE key = 'schema_version';" + +static const struct db_init_query db_upgrade_v13_queries[] = + { + { U_V13_DROP_IDX_PATH, "drop index path table files" }, + { U_V13_DROP_IDX_TS, "drop index titlesort table files" }, + { U_V13_DROP_IDX_AS, "drop index artistsort table files" }, + { U_V13_DROP_IDX_BS, "drop index albumsort table files" }, + + { U_V13_IDX_RESCAN, "create rescan index" }, + { U_V13_IDX_SONGALBUMID, "create songalbumid index" }, + { U_V13_IDX_STATEMKINDSAI, "create state/mkind/sai index" }, + { U_V13_IDX_ARTIST, "create artist index" }, + { U_V13_IDX_ALBUMARTIST, "create album_artist index" }, + { U_V13_IDX_COMPOSER, "create composer index" }, + { U_V13_IDX_TITLE, "create title index" }, + { U_V13_IDX_ALBUM, "create album index" }, + + { U_V13_IDX_GRP_TYPE_PERSIST, "create groups type/persistentid index" }, + + { U_V13_IDX_PL_PATH, "create playlist path index" }, + { U_V13_IDX_PL_DISABLED, "create playlist state index" }, + + { U_V13_PL2, "update default smart playlist 'Music'" }, + { U_V13_PL3, "update default smart playlist 'Movies'" }, + { U_V13_PL4, "update default smart playlist 'TV Shows'" }, + + { U_V13_SCVER, "set schema_version to 13" }, + }; static int db_check_version(void) @@ -4688,6 +4774,13 @@ db_check_version(void) if (ret < 0) return -1; + /* FALLTHROUGH */ + + case 12: + ret = db_generic_upgrade(db_upgrade_v13_queries, sizeof(db_upgrade_v13_queries) / sizeof(db_upgrade_v13_queries[0])); + if (ret < 0) + return -1; + break; default: From 9f06848d4374ad30b815439960636da0cd013c71 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 18:37:16 +0200 Subject: [PATCH 23/26] Reply to update requests periodically to avoid 30-minute iTunes timeout Craig Markwardt found out that the 30-minute timeout in iTunes was caused by the lack of reply to update requests. We now send out replies every 5 minutes, avoiding the timeout and disconnection. Thanks to Craig for digging into this, producing code to demonstrate the fix and trying out a few more ideas for update support beyond this fix. --- src/httpd_daap.c | 99 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 11 deletions(-) diff --git a/src/httpd_daap.c b/src/httpd_daap.c index 3017b3bf..8fe60092 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -59,6 +59,8 @@ extern struct event_base *evbase_httpd; /* Session timeout in seconds */ #define DAAP_SESSION_TIMEOUT 1800 +/* Update requests refresh interval in seconds */ +#define DAAP_UPDATE_REFRESH 300 struct uri_map { @@ -76,6 +78,9 @@ struct daap_session { struct daap_update_request { struct evhttp_request *req; + /* Refresh tiemout */ + struct event timeout; + struct daap_update_request *next; }; @@ -98,6 +103,7 @@ static avl_tree_t *daap_sessions; static int next_session_id; /* Update requests */ +static int current_rev; static struct daap_update_request *update_requests; @@ -248,18 +254,10 @@ daap_session_find(struct evhttp_request *req, struct evkeyvalq *query, struct ev /* Update requests helpers */ static void -update_fail_cb(struct evhttp_connection *evcon, void *arg) +update_remove(struct daap_update_request *ur) { - struct daap_update_request *ur; struct daap_update_request *p; - ur = (struct daap_update_request *)arg; - - DPRINTF(E_DBG, L_DAAP, "Update request: client closed connection\n"); - - if (ur->req->evcon) - evhttp_connection_set_closecb(ur->req->evcon, NULL, NULL); - if (ur == update_requests) update_requests = ur->next; else @@ -275,10 +273,70 @@ update_fail_cb(struct evhttp_connection *evcon, void *arg) p->next = ur->next; } +} +static void +update_free(struct daap_update_request *ur) +{ + if (event_initialized(&ur->timeout)) + evtimer_del(&ur->timeout); free(ur); } +static void +update_refresh_cb(int fd, short event, void *arg) +{ + struct daap_update_request *ur; + struct evbuffer *evbuf; + int ret; + + ur = (struct daap_update_request *)arg; + + evbuf = evbuffer_new(); + if (!evbuf) + { + DPRINTF(E_LOG, L_DAAP, "Could not allocate evbuffer for DAAP update data\n"); + + return; + } + + ret = evbuffer_expand(evbuf, 32); + if (ret < 0) + { + DPRINTF(E_LOG, L_DAAP, "Could not expand evbuffer for DAAP update data\n"); + + return; + } + + /* Send back current revision */ + dmap_add_container(evbuf, "mupd", 24); + dmap_add_int(evbuf, "mstt", 200); /* 12 */ + dmap_add_int(evbuf, "musr", current_rev); /* 12 */ + + evhttp_connection_set_closecb(ur->req->evcon, NULL, NULL); + + httpd_send_reply(ur->req, HTTP_OK, "OK", evbuf); + + update_remove(ur); + update_free(ur); +} + +static void +update_fail_cb(struct evhttp_connection *evcon, void *arg) +{ + struct daap_update_request *ur; + + ur = (struct daap_update_request *)arg; + + DPRINTF(E_DBG, L_DAAP, "Update request: client closed connection\n"); + + if (ur->req->evcon) + evhttp_connection_set_closecb(ur->req->evcon, NULL, NULL); + + update_remove(ur); + update_free(ur); +} + /* DAAP sort headers helpers */ static struct sort_ctx * @@ -780,10 +838,10 @@ daap_reply_logout(struct evhttp_request *req, struct evbuffer *evbuf, char **uri static void daap_reply_update(struct evhttp_request *req, struct evbuffer *evbuf, char **uri, struct evkeyvalq *query) { + struct timeval tv; struct daap_session *s; struct daap_update_request *ur; const char *param; - int current_rev = 2; int reqd_rev; int ret; @@ -839,6 +897,24 @@ daap_reply_update(struct evhttp_request *req, struct evbuffer *evbuf, char **uri dmap_send_error(req, "mupd", "Out of memory"); return; } + memset(ur, 0, sizeof(struct daap_update_request)); + + evtimer_set(&ur->timeout, update_refresh_cb, ur); + event_base_set(evbase_httpd, &ur->timeout); + + evutil_timerclear(&tv); + tv.tv_sec = DAAP_UPDATE_REFRESH; + + ret = evtimer_add(&ur->timeout, &tv); + if (ret < 0) + { + DPRINTF(E_LOG, L_DAAP, "Could not add update timeout event\n"); + + dmap_send_error(req, "mupd", "Could not register timer"); + + update_free(ur); + return; + } /* NOTE: we may need to keep reqd_rev in there too */ ur->req = req; @@ -2428,6 +2504,7 @@ daap_init(void) int ret; next_session_id = 100; /* gotta start somewhere, right? */ + current_rev = 2; update_requests = NULL; for (i = 0; daap_handlers[i].handler; i++) @@ -2480,6 +2557,6 @@ daap_deinit(void) evhttp_connection_free(ur->req->evcon); } - free(ur); + update_free(ur); } } From 4be0d86aa0d8592cf9dd16be6345376cff5b3870 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sat, 10 Sep 2011 18:44:37 +0200 Subject: [PATCH 24/26] Revert "Disable session expiration" This reverts commit c70caad87e12357d7d912113f6a06b08d781afd1. --- src/httpd_daap.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/httpd_daap.c b/src/httpd_daap.c index 8fe60092..650b9d68 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -155,14 +155,10 @@ daap_session_timeout_cb(int fd, short what, void *arg) static struct daap_session * daap_session_register(void) { -#if 0 struct timeval tv; -#endif struct daap_session *s; avl_node_t *node; -#if 0 int ret; -#endif s = (struct daap_session *)malloc(sizeof(struct daap_session)); if (!s) @@ -189,14 +185,12 @@ daap_session_register(void) return NULL; } -#if 0 evutil_timerclear(&tv); tv.tv_sec = DAAP_SESSION_TIMEOUT; ret = evtimer_add(&s->timeout, &tv); if (ret < 0) DPRINTF(E_LOG, L_DAAP, "Could not add session timeout event for session %d\n", s->id); -#endif /* 0 */ return s; } @@ -205,9 +199,7 @@ struct daap_session * daap_session_find(struct evhttp_request *req, struct evkeyvalq *query, struct evbuffer *evbuf) { struct daap_session needle; -#if 0 struct timeval tv; -#endif struct daap_session *s; avl_node_t *node; const char *param; @@ -233,7 +225,6 @@ daap_session_find(struct evhttp_request *req, struct evkeyvalq *query, struct ev s = (struct daap_session *)node->item; -#if 0 event_del(&s->timeout); evutil_timerclear(&tv); @@ -242,7 +233,6 @@ daap_session_find(struct evhttp_request *req, struct evkeyvalq *query, struct ev ret = evtimer_add(&s->timeout, &tv); if (ret < 0) DPRINTF(E_LOG, L_DAAP, "Could not add session timeout event for session %d\n", s->id); -#endif /* 0 */ return s; @@ -656,7 +646,7 @@ daap_reply_server_info(struct evhttp_request *req, struct evbuffer *evbuf, char passwd = cfg_getstr(lib, "password"); name = cfg_getstr(lib, "name"); - len = 136 + strlen(name); + len = 157 + strlen(name); ret = evbuffer_expand(evbuf, len); if (ret < 0) @@ -691,10 +681,8 @@ daap_reply_server_info(struct evhttp_request *req, struct evbuffer *evbuf, char dmap_add_int(evbuf, "apro", apro); /* 12 */ dmap_add_string(evbuf, "minm", name); /* 8 + strlen(name) */ -#if 0 dmap_add_int(evbuf, "mstm", DAAP_SESSION_TIMEOUT); /* 12 */ dmap_add_char(evbuf, "msal", 1); /* 9 */ -#endif dmap_add_char(evbuf, "mslr", 1); /* 9 */ dmap_add_char(evbuf, "msau", (passwd) ? 2 : 0); /* 9 */ From 078570954e76a9d520edc151058ea6d47484d414 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 11 Sep 2011 15:45:42 +0200 Subject: [PATCH 25/26] ChangeLog for forked-daapd 0.19 --- ChangeLog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2266c1ac..5d27ffd4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,12 @@ ChangeLog for forked-daapd -------------------------- +version 0.19: + - more libav 0.7 updates. + - database speedups. + - fix for iTunes 30-minute timeout. + - fixes, big and small. + version 0.18: - add config knob for ALSA mixer channel name. - do not elevate privileges for reopening the log file; log file From f7d7dfc9195497cfe65f980ad6cacda1b2970ea7 Mon Sep 17 00:00:00 2001 From: Julien BLACHE Date: Sun, 11 Sep 2011 15:46:25 +0200 Subject: [PATCH 26/26] Bump version to 0.19 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 5c53c8e4..4b59869b 100644 --- a/configure.in +++ b/configure.in @@ -3,7 +3,7 @@ dnl Process this file with autoconf to produce a configure script. AC_INIT(config.h.in) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADER(config.h) -AM_INIT_AUTOMAKE(forked-daapd, 0.18) +AM_INIT_AUTOMAKE(forked-daapd, 0.19) AC_USE_SYSTEM_EXTENSIONS