From 7703a997c4fefb098053f27473bdb6ed690b75fb Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 27 Dec 2015 07:16:50 +0100 Subject: [PATCH] [db/filescanner/spotify] Replace filelist-view with separate directories-table to increase the performance of the mpd command 'lsinfo' --- src/db.c | 540 ++++++++++++++++++++++++++----------- src/db.h | 66 +++-- src/db_upgrade.c | 268 ++++++++++++++++++ src/filescanner.c | 197 +++++++++++--- src/filescanner.h | 6 +- src/filescanner_playlist.c | 14 +- src/filescanner_smartpl.c | 4 +- src/mpd.c | 130 +++++---- src/spotify.c | 49 +++- 9 files changed, 1008 insertions(+), 266 deletions(-) diff --git a/src/db.c b/src/db.c index 607a0a74..6a0bd982 100644 --- a/src/db.c +++ b/src/db.c @@ -137,6 +137,7 @@ static const struct col_type_map mfi_cols_map[] = { mfi_offsetof(composer_sort), DB_TYPE_STRING }, { mfi_offsetof(album_artist_sort), DB_TYPE_STRING }, { mfi_offsetof(virtual_path), DB_TYPE_STRING }, + { mfi_offsetof(directory_id), DB_TYPE_INT }, }; /* This list must be kept in sync with @@ -156,6 +157,7 @@ static const struct col_type_map pli_cols_map[] = { pli_offsetof(special_id), DB_TYPE_INT }, { pli_offsetof(virtual_path), DB_TYPE_STRING }, { pli_offsetof(parent_id), DB_TYPE_INT }, + { pli_offsetof(directory_id), DB_TYPE_INT }, /* items is computed on the fly */ }; @@ -224,6 +226,7 @@ static const ssize_t dbmfi_cols_map[] = dbmfi_offsetof(composer_sort), dbmfi_offsetof(album_artist_sort), dbmfi_offsetof(virtual_path), + dbmfi_offsetof(directory_id), }; /* This list must be kept in sync with @@ -243,6 +246,7 @@ static const ssize_t dbpli_cols_map[] = dbpli_offsetof(special_id), dbpli_offsetof(virtual_path), dbpli_offsetof(parent_id), + dbpli_offsetof(directory_id), /* items is computed on the fly */ }; @@ -330,18 +334,6 @@ db_escape_string(const char *str) return ret; } -void -free_fi(struct filelist_info *fi, int content_only) -{ - if (fi->virtual_path) - free(fi->virtual_path); - - if (!content_only) - free(fi); - else - memset(fi, 0, sizeof(struct filelist_info)); -} - void free_pi(struct pairing_info *pi, int content_only) { @@ -496,6 +488,18 @@ free_pli(struct playlist_info *pli, int content_only) memset(pli, 0, sizeof(struct playlist_info)); } +void +free_di(struct directory_info *di, int content_only) +{ + if (di->virtual_path) + free(di->virtual_path); + + if (!content_only) + free(di); + else + memset(di, 0, sizeof(struct directory_info)); +} + /* Unlock notification support */ static void @@ -708,12 +712,13 @@ db_purge_cruft(time_t ref) char *errmsg; int i; int ret; - char *queries[3] = { NULL, NULL, NULL }; - char *queries_tmpl[3] = + char *queries[4] = { NULL, NULL, NULL, NULL }; + char *queries_tmpl[4] = { "DELETE FROM playlistitems WHERE playlistid IN (SELECT id FROM playlists p WHERE p.type <> %d AND p.db_timestamp < %" PRIi64 ");", "DELETE FROM playlists WHERE type <> %d AND db_timestamp < %" PRIi64 ";", - "DELETE FROM files WHERE -1 <> %d AND db_timestamp < %" PRIi64 ";" + "DELETE FROM files WHERE -1 <> %d AND db_timestamp < %" PRIi64 ";", + "DELETE FROM directories WHERE id > 1 AND -1 <> %d AND db_timestamp < %" PRIi64 ";" }; if (sizeof(queries) != sizeof(queries_tmpl)) @@ -1950,89 +1955,6 @@ db_query_fetch_string_sort(struct query_params *qp, char **string, char **sortst return 0; } -/* Filelist */ - -int -db_mpd_start_query_filelist(struct query_params *qp, char *parentpath) -{ - char *query; - int ret; - - query = sqlite3_mprintf( - "SELECT " - " CASE WHEN daap_charindex('/', virtual_path, LENGTH(%Q)+1) = 0 " - " THEN " - " virtual_path " - " ELSE " - " daap_leftstr(virtual_path, daap_charindex('/', virtual_path, LENGTH(%Q)+1)-1) " - " END AS path, " - " MAX(time_modified), " - " CASE WHEN daap_charindex('/', virtual_path, LENGTH(%Q)+1) = 0 " - " THEN " - " type " - " ELSE " - " 2 " - " END AS ftype " - "FROM filelist " - "WHERE virtual_path LIKE '%q%%' " - "GROUP BY ftype, path " - "ORDER BY ftype, path;", parentpath, parentpath, parentpath, parentpath); - - if (!query) - { - DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); - return -1; - } - - DPRINTF(E_DBG, L_DB, "Starting query '%s'\n", query); - - ret = db_blocking_prepare_v2(query, -1, &qp->stmt, NULL); - if (ret != SQLITE_OK) - { - DPRINTF(E_LOG, L_DB, "Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); - - sqlite3_free(query); - return -1; - } - - sqlite3_free(query); - - return 0; -} - -int -db_mpd_query_fetch_filelist(struct query_params *qp, struct filelist_info *fi) -{ - int ret; - - memset(fi, 0, sizeof(struct filelist_info)); - - if (!qp->stmt) - { - DPRINTF(E_LOG, L_DB, "Query not started!\n"); - return -1; - } - - ret = db_blocking_step(qp->stmt); - if (ret == SQLITE_DONE) - { - DPRINTF(E_DBG, L_DB, "End of query results\n"); - fi->virtual_path = NULL; - return 0; - } - else if (ret != SQLITE_ROW) - { - DPRINTF(E_LOG, L_DB, "Could not step: %s\n", sqlite3_errmsg(hdl)); - return -1; - } - - fi->virtual_path = strdup((char *)sqlite3_column_text(qp->stmt, 0)); - fi->time_modified = sqlite3_column_int(qp->stmt, 1); - fi->type = sqlite3_column_int(qp->stmt, 2); - - return 0; -} - /* Files */ int @@ -2283,6 +2205,7 @@ db_file_id_bymatch(char *path) #undef Q_TMPL } +//TODO [cleanup] unused function(?) int db_file_id_byfilebase(char *filename, char *base) { @@ -2609,8 +2532,8 @@ db_file_add(struct media_file_info *mfi) " codectype, idx, has_video, contentrating, bits_per_sample, album_artist," \ " media_kind, tv_series_name, tv_episode_num_str, tv_network_name, tv_episode_sort, tv_season_num, " \ " songartistid, songalbumid, " \ - " title_sort, artist_sort, album_sort, composer_sort, album_artist_sort, virtual_path" \ - " ) " \ + " title_sort, artist_sort, album_sort, composer_sort, album_artist_sort, virtual_path," \ + " directory_id) " \ " VALUES (NULL, '%q', '%q', TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), %Q, TRIM(%Q)," \ " TRIM(%Q), TRIM(%Q), TRIM(%Q), %Q, %d, %d, %d, %" PRIi64 ", %d, %d," \ " %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d," \ @@ -2618,7 +2541,7 @@ db_file_add(struct media_file_info *mfi) " %Q, %d, %d, %d, %d, TRIM(%Q)," \ " %d, TRIM(%Q), TRIM(%Q), TRIM(%Q), %d, %d," \ " daap_songalbumid(LOWER(TRIM(%Q)), ''), daap_songalbumid(LOWER(TRIM(%Q)), LOWER(TRIM(%Q))), " \ - " TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q));" + " TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), TRIM(%Q), %d);" char *query; char *errmsg; @@ -2653,7 +2576,7 @@ db_file_add(struct media_file_info *mfi) mfi->media_kind, mfi->tv_series_name, mfi->tv_episode_num_str, mfi->tv_network_name, mfi->tv_episode_sort, mfi->tv_season_num, mfi->album_artist, mfi->album_artist, mfi->album, mfi->title_sort, mfi->artist_sort, mfi->album_sort, - mfi->composer_sort, mfi->album_artist_sort, mfi->virtual_path); + mfi->composer_sort, mfi->album_artist_sort, mfi->virtual_path, mfi->directory_id); if (!query) { @@ -2698,10 +2621,9 @@ db_file_update(struct media_file_info *mfi) " tv_network_name = TRIM(%Q), tv_episode_sort = %d, tv_season_num = %d," \ " songartistid = daap_songalbumid(LOWER(TRIM(%Q)), ''), songalbumid = daap_songalbumid(LOWER(TRIM(%Q)), LOWER(TRIM(%Q)))," \ " title_sort = TRIM(%Q), artist_sort = TRIM(%Q), album_sort = TRIM(%Q), composer_sort = TRIM(%Q), album_artist_sort = TRIM(%Q)," \ - " virtual_path = TRIM(%Q)" \ + " virtual_path = TRIM(%Q), directory_id = %d" \ " WHERE id = %d;" -// struct media_file_info *oldmfi; char *query; char *errmsg; int ret; @@ -2712,18 +2634,6 @@ db_file_update(struct media_file_info *mfi) return -1; } - /* - oldmfi = db_file_fetch_byid(mfi->id); - - if (!oldmfi) - { - DPRINTF(E_WARN, L_DB, "File with id '%d' does not exist\n", mfi->id); - return -1; - } - - free_mfi(oldmfi, 0); - */ - mfi->db_timestamp = (uint64_t)time(NULL); if (mfi->time_modified == 0) @@ -2743,7 +2653,7 @@ db_file_update(struct media_file_info *mfi) mfi->tv_network_name, mfi->tv_episode_sort, mfi->tv_season_num, mfi->album_artist, mfi->album_artist, mfi->album, mfi->title_sort, mfi->artist_sort, mfi->album_sort, - mfi->composer_sort, mfi->album_artist_sort, mfi->virtual_path, + mfi->composer_sort, mfi->album_artist_sort, mfi->virtual_path, mfi->directory_id, mfi->id); if (!query) @@ -2830,15 +2740,20 @@ db_file_delete_bypath(char *path) void db_file_disable_bypath(char *path, char *strip, uint32_t cookie) { -#define Q_TMPL "UPDATE files SET path = substr(path, %d), disabled = %" PRIi64 " WHERE path = '%q';" +#define Q_TMPL "UPDATE files SET path = substr(path, %d), virtual_path = substr(virtual_path, %d), disabled = %" PRIi64 " WHERE path = '%q';" char *query; int64_t disabled; int striplen; + int striplenvpath; disabled = (cookie != 0) ? cookie : INOTIFY_FAKE_COOKIE; striplen = strlen(strip) + 1; + if (strlen(strip) > 0) + striplenvpath = strlen(strip) + strlen("/file:/"); + else + striplenvpath = 0; - query = sqlite3_mprintf(Q_TMPL, striplen, disabled, path); + query = sqlite3_mprintf(Q_TMPL, striplen, striplenvpath, disabled, path); db_query_run(query, 1, 1); #undef Q_TMPL @@ -2847,15 +2762,20 @@ db_file_disable_bypath(char *path, char *strip, uint32_t cookie) void db_file_disable_bymatch(char *path, char *strip, uint32_t cookie) { -#define Q_TMPL "UPDATE files SET path = substr(path, %d), disabled = %" PRIi64 " WHERE path LIKE '%q/%%';" +#define Q_TMPL "UPDATE files SET path = substr(path, %d), virtual_path = substr(virtual_path, %d), disabled = %" PRIi64 " WHERE path LIKE '%q/%%';" char *query; int64_t disabled; int striplen; + int striplenvpath; disabled = (cookie != 0) ? cookie : INOTIFY_FAKE_COOKIE; striplen = strlen(strip) + 1; + if (strlen(strip) > 0) + striplenvpath = strlen(strip) + strlen("/file:/"); + else + striplenvpath = 0; - query = sqlite3_mprintf(Q_TMPL, striplen, disabled, path); + query = sqlite3_mprintf(Q_TMPL, striplen, striplenvpath, disabled, path); db_query_run(query, 1, 1); #undef Q_TMPL @@ -2864,11 +2784,11 @@ db_file_disable_bymatch(char *path, char *strip, uint32_t cookie) int db_file_enable_bycookie(uint32_t cookie, char *path) { -#define Q_TMPL "UPDATE files SET path = '%q' || path, disabled = 0 WHERE disabled = %" PRIi64 ";" +#define Q_TMPL "UPDATE files SET path = '%q' || path, virtual_path = '/file:%q' || virtual_path, disabled = 0 WHERE disabled = %" PRIi64 ";" char *query; int ret; - query = sqlite3_mprintf(Q_TMPL, path, (int64_t)cookie); + query = sqlite3_mprintf(Q_TMPL, path, path, (int64_t)cookie); ret = db_query_run(query, 1, 1); @@ -2876,6 +2796,21 @@ db_file_enable_bycookie(uint32_t cookie, char *path) #undef Q_TMPL } +int +db_file_update_directoryid(char *path, int dir_id) +{ +#define Q_TMPL "UPDATE files SET directory_id = %d WHERE path = %Q;" + char *query; + int ret; + + query = sqlite3_mprintf(Q_TMPL, dir_id, path); + + ret = db_query_run(query, 1, 0); + + return ((ret < 0) ? -1 : sqlite3_changes(hdl)); +#undef Q_TMPL +} + /* Playlists */ int @@ -3249,8 +3184,8 @@ int db_pl_add(struct playlist_info *pli, int *id) { #define QDUP_TMPL "SELECT COUNT(*) FROM playlists p WHERE p.title = TRIM(%Q) AND p.path = '%q';" -#define QADD_TMPL "INSERT INTO playlists (title, type, query, db_timestamp, disabled, path, idx, special_id, parent_id, virtual_path)" \ - " VALUES (TRIM(%Q), %d, '%q', %" PRIi64 ", %d, '%q', %d, %d, %d, '%q');" +#define QADD_TMPL "INSERT INTO playlists (title, type, query, db_timestamp, disabled, path, idx, special_id, parent_id, virtual_path, directory_id)" \ + " VALUES (TRIM(%Q), %d, '%q', %" PRIi64 ", %d, '%q', %d, %d, %d, '%q', %d);" char *query; char *errmsg; int ret; @@ -3276,7 +3211,7 @@ db_pl_add(struct playlist_info *pli, int *id) /* Add */ query = sqlite3_mprintf(QADD_TMPL, pli->title, pli->type, pli->query, (int64_t)time(NULL), pli->disabled, STR(pli->path), - pli->index, pli->special_id, pli->parent_id, pli->virtual_path); + pli->index, pli->special_id, pli->parent_id, pli->virtual_path, pli->directory_id); if (!query) { @@ -3341,14 +3276,14 @@ int db_pl_update(struct playlist_info *pli) { #define Q_TMPL "UPDATE playlists SET title = TRIM(%Q), type = %d, query = '%q', db_timestamp = %" PRIi64 ", disabled = %d, " \ - " path = '%q', idx = %d, special_id = %d, parent_id = %d, virtual_path = '%q' " \ + " path = '%q', idx = %d, special_id = %d, parent_id = %d, virtual_path = '%q', directory_id = %d " \ " WHERE id = %d;" char *query; int ret; query = sqlite3_mprintf(Q_TMPL, pli->title, pli->type, pli->query, (int64_t)time(NULL), pli->disabled, STR(pli->path), - pli->index, pli->special_id, pli->parent_id, pli->virtual_path, pli->id); + pli->index, pli->special_id, pli->parent_id, pli->virtual_path, pli->directory_id, pli->id); ret = db_query_run(query, 1, 0); @@ -3403,15 +3338,20 @@ db_pl_delete_bypath(char *path) void db_pl_disable_bypath(char *path, char *strip, uint32_t cookie) { -#define Q_TMPL "UPDATE playlists SET path = substr(path, %d), disabled = %" PRIi64 " WHERE path = '%q';" +#define Q_TMPL "UPDATE playlists SET path = substr(path, %d), virtual_path = substr(virtual_path, %d), disabled = %" PRIi64 " WHERE path = '%q';" char *query; int64_t disabled; int striplen; + int striplenvpath; disabled = (cookie != 0) ? cookie : INOTIFY_FAKE_COOKIE; striplen = strlen(strip) + 1; + if (strlen(strip) > 0) + striplenvpath = strlen(strip) + strlen("/file:/"); + else + striplenvpath = 0; - query = sqlite3_mprintf(Q_TMPL, striplen, disabled, path); + query = sqlite3_mprintf(Q_TMPL, striplen, striplenvpath, disabled, path); db_query_run(query, 1, 0); #undef Q_TMPL @@ -3420,15 +3360,20 @@ db_pl_disable_bypath(char *path, char *strip, uint32_t cookie) void db_pl_disable_bymatch(char *path, char *strip, uint32_t cookie) { -#define Q_TMPL "UPDATE playlists SET path = substr(path, %d), disabled = %" PRIi64 " WHERE path LIKE '%q/%%';" +#define Q_TMPL "UPDATE playlists SET path = substr(path, %d), virtual_path = substr(virtual_path, %d), disabled = %" PRIi64 " WHERE path LIKE '%q/%%';" char *query; int64_t disabled; int striplen; + int striplenvpath; disabled = (cookie != 0) ? cookie : INOTIFY_FAKE_COOKIE; striplen = strlen(strip) + 1; + if (strlen(strip) > 0) + striplenvpath = strlen(strip) + strlen("/file:/"); + else + striplenvpath = 0; - query = sqlite3_mprintf(Q_TMPL, striplen, disabled, path); + query = sqlite3_mprintf(Q_TMPL, striplen, striplenvpath, disabled, path); db_query_run(query, 1, 0); #undef Q_TMPL @@ -3437,11 +3382,11 @@ db_pl_disable_bymatch(char *path, char *strip, uint32_t cookie) int db_pl_enable_bycookie(uint32_t cookie, char *path) { -#define Q_TMPL "UPDATE playlists SET path = '%q' || path, disabled = 0 WHERE disabled = %" PRIi64 ";" +#define Q_TMPL "UPDATE playlists SET path = '%q' || path, virtual_path = '/file:%q' || virtual_path, disabled = 0 WHERE disabled = %" PRIi64 ";" char *query; int ret; - query = sqlite3_mprintf(Q_TMPL, path, (int64_t)cookie); + query = sqlite3_mprintf(Q_TMPL, path, path, (int64_t)cookie); ret = db_query_run(query, 1, 0); @@ -3569,6 +3514,275 @@ db_group_persistentid_byid(int id, int64_t *persistentid) } +/* Directories */ +int +db_directory_id_byvirtualpath(char *virtual_path) +{ +#define Q_TMPL "SELECT d.id FROM directories d WHERE d.virtual_path = '%q';" + char *query; + int ret; + + query = sqlite3_mprintf(Q_TMPL, virtual_path); + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + + return 0; + } + + ret = db_file_id_byquery(query); + + sqlite3_free(query); + + return ret; + +#undef Q_TMPL +} + +int +db_directory_enum_start(struct directory_enum *de) +{ +#define Q_TMPL "SELECT * FROM directories WHERE disabled = 0 AND parent_id = %d ORDER BY virtual_path;" + char *query; + int ret; + + de->stmt = NULL; + + query = sqlite3_mprintf(Q_TMPL, de->parent_id); + + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + + return -1; + } + + DPRINTF(E_DBG, L_DB, "Starting enum '%s'\n", query); + + ret = db_blocking_prepare_v2(query, -1, &de->stmt, NULL); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); + + sqlite3_free(query); + return -1; + } + + sqlite3_free(query); + + return 0; + +#undef Q_TMPL +} + +int +db_directory_enum_fetch(struct directory_enum *de, struct directory_info *di) +{ + uint64_t disabled; + int ret; + + memset(di, 0, sizeof(struct directory_info)); + + if (!de->stmt) + { + DPRINTF(E_LOG, L_DB, "Directory enum not started!\n"); + return -1; + } + + ret = db_blocking_step(de->stmt); + if (ret == SQLITE_DONE) + { + DPRINTF(E_DBG, L_DB, "End of directory enum results\n"); + return 0; + } + else if (ret != SQLITE_ROW) + { + DPRINTF(E_LOG, L_DB, "Could not step: %s\n", sqlite3_errmsg(hdl)); + return -1; + } + + di->id = sqlite3_column_int(de->stmt, 0); + di->virtual_path = (char *)sqlite3_column_text(de->stmt, 1); + di->db_timestamp = sqlite3_column_int(de->stmt, 2); + disabled = sqlite3_column_int64(de->stmt, 3); + di->disabled = (disabled != 0); + di->parent_id = sqlite3_column_int(de->stmt, 4); + + return 0; +} + +void +db_directory_enum_end(struct directory_enum *de) +{ + if (!de->stmt) + return; + + sqlite3_finalize(de->stmt); + de->stmt = NULL; +} + +static int +db_directory_add(struct directory_info *di, int *id) +{ +#define QADD_TMPL "INSERT INTO directories (virtual_path, db_timestamp, disabled, parent_id)" \ + " VALUES (TRIM(%Q), %d, %d, %d);" + + char *query; + char *errmsg; + int ret; + + query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id); + + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + return -1; + } + + DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); + + ret = db_exec(query, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Query error: %s\n", errmsg); + + sqlite3_free(errmsg); + sqlite3_free(query); + return -1; + } + + sqlite3_free(query); + + *id = (int)sqlite3_last_insert_rowid(hdl); + if (*id == 0) + { + DPRINTF(E_LOG, L_DB, "Successful insert but no last_insert_rowid!\n"); + return -1; + } + + DPRINTF(E_DBG, L_DB, "Added directory %s with id %d\n", di->virtual_path, *id); + + return 0; + +#undef QADD_TMPL +} + +static int +db_directory_update(struct directory_info *di) +{ +#define QADD_TMPL "UPDATE directories SET virtual_path = TRIM(%Q), db_timestamp = %d, disabled = %d, parent_id = %d" \ + " WHERE id = %d;" + char *query; + char *errmsg; + int ret; + + /* Add */ + query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id, di->id); + + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + return -1; + } + + DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); + + ret = db_exec(query, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Query error: %s\n", errmsg); + + sqlite3_free(errmsg); + sqlite3_free(query); + return -1; + } + + sqlite3_free(query); + + DPRINTF(E_DBG, L_DB, "Updated directory %s with id %d\n", di->virtual_path, di->id); + + return 0; + +#undef QADD_TMPL +} + +int +db_directory_addorupdate(char *virtual_path, int disabled, int parent_id) +{ + struct directory_info di; + int id; + int ret; + + id = db_directory_id_byvirtualpath(virtual_path); + + di.id = id; + di.parent_id = parent_id; + di.virtual_path = virtual_path; + di.disabled = disabled; + di.db_timestamp = (uint64_t)time(NULL); + + if (di.id == 0) + ret = db_directory_add(&di, &id); + else + ret = db_directory_update(&di); + + if (ret < 0 || id <= 0) + { + DPRINTF(E_LOG, L_DB, "Insert or update of directory failed '%s'\n", virtual_path); + return -1; + } + + return id; +} + +void +db_directory_ping_bymatch(char *path) +{ +#define Q_TMPL_DIR "UPDATE directories SET db_timestamp = %" PRIi64 " WHERE virtual_path = '/file:%q' OR virtual_path LIKE '/file:%q/%%';" + char *query; + + query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), path, path); + + db_query_run(query, 1, 1); +#undef Q_TMPL_DIR +} + +void +db_directory_disable_bymatch(char *path, char *strip, uint32_t cookie) +{ +#define Q_TMPL "UPDATE directories SET virtual_path = substr(virtual_path, %d), disabled = %" PRIi64 " WHERE virtual_path = '/file:%q' OR virtual_path LIKE '/file:%q/%%';" + char *query; + int64_t disabled; + int striplen; + + disabled = (cookie != 0) ? cookie : INOTIFY_FAKE_COOKIE; + if (strlen(strip) > 0) + striplen = strlen(strip) + strlen("/file:/"); + else + striplen = 0; + + query = sqlite3_mprintf(Q_TMPL, striplen, disabled, path, path, path); + + db_query_run(query, 1, 1); +#undef Q_TMPL +} + +int +db_directory_enable_bycookie(uint32_t cookie, char *path) +{ +#define Q_TMPL "UPDATE directories SET virtual_path = '/file:%q' || virtual_path, disabled = 0 WHERE disabled = %" PRIi64 ";" + char *query; + int ret; + + query = sqlite3_mprintf(Q_TMPL, path, (int64_t)cookie); + + ret = db_query_run(query, 1, 0); + + return ((ret < 0) ? -1 : sqlite3_changes(hdl)); +#undef Q_TMPL +} + + /* Remotes */ static int db_pairing_delete_byremote(char *remote_id) @@ -4635,7 +4849,8 @@ db_perthread_deinit(void) " album_sort VARCHAR(1024) DEFAULT NULL COLLATE DAAP," \ " composer_sort VARCHAR(1024) DEFAULT NULL COLLATE DAAP," \ " album_artist_sort VARCHAR(1024) DEFAULT NULL COLLATE DAAP," \ - " virtual_path VARCHAR(4096) DEFAULT NULL" \ + " virtual_path VARCHAR(4096) DEFAULT NULL," \ + " directory_id INTEGER DEFAULT 0" \ ");" #define T_PL \ @@ -4650,7 +4865,8 @@ db_perthread_deinit(void) " idx INTEGER NOT NULL," \ " special_id INTEGER DEFAULT 0," \ " virtual_path VARCHAR(4096)," \ - " parent_id INTEGER DEFAULT 0" \ + " parent_id INTEGER DEFAULT 0," \ + " directory_id INTEGER DEFAULT 0" \ ");" #define T_PLITEMS \ @@ -4690,16 +4906,14 @@ db_perthread_deinit(void) " path VARCHAR(4096) NOT NULL" \ ");" -#define V_FILELIST \ - "CREATE VIEW IF NOT EXISTS filelist as" \ - " SELECT " \ - " virtual_path, time_modified, 3 as type " \ - " FROM files WHERE disabled = 0" \ - " UNION " \ - " SELECT " \ - " virtual_path, db_timestamp, 1 as type " \ - " FROM playlists where disabled = 0 AND type IN (2, 3)" \ - ";" +#define T_DIRECTORIES \ + "CREATE TABLE IF NOT EXISTS directories (" \ + " id INTEGER PRIMARY KEY NOT NULL," \ + " virtual_path VARCHAR(4096) NOT NULL," \ + " db_timestamp INTEGER DEFAULT 0," \ + " disabled INTEGER DEFAULT 0," \ + " parent_id INTEGER DEFAULT 0" \ + ");" #define TRG_GROUPS_INSERT_FILES \ "CREATE TRIGGER update_groups_new_file AFTER INSERT ON files FOR EACH ROW" \ @@ -4745,16 +4959,21 @@ db_perthread_deinit(void) " VALUES(8, 'Purchased', 0, 'media_kind = 1024', 0, '', 0, 8);" */ + +#define Q_DIR1 \ + "INSERT INTO directories (id, virtual_path, db_timestamp, disabled, parent_id)" \ + " VALUES (1, '/', 0, 0, 0);" + /* Rule of thumb: Will the current version of forked-daapd work with the new * version of the database? If yes, then it is a minor upgrade, if no, then it * is a major upgrade. In other words minor version upgrades permit downgrading * forked-daapd after the database was upgraded. */ -#define SCHEMA_VERSION_MAJOR 18 -#define SCHEMA_VERSION_MINOR 01 +#define SCHEMA_VERSION_MAJOR 19 +#define SCHEMA_VERSION_MINOR 00 #define Q_SCVER_MAJOR \ - "INSERT INTO admin (key, value) VALUES ('schema_version_major', '18');" + "INSERT INTO admin (key, value) VALUES ('schema_version_major', '19');" #define Q_SCVER_MINOR \ - "INSERT INTO admin (key, value) VALUES ('schema_version_minor', '01');" + "INSERT INTO admin (key, value) VALUES ('schema_version_minor', '00');" struct db_init_query { char *query; @@ -4771,8 +4990,7 @@ static const struct db_init_query db_init_table_queries[] = { T_PAIRINGS, "create table pairings" }, { T_SPEAKERS, "create table speakers" }, { T_INOTIFY, "create table inotify" }, - - { V_FILELIST, "create view filelist" }, + { T_DIRECTORIES, "create table directories" }, { TRG_GROUPS_INSERT_FILES, "create trigger update_groups_new_file" }, { TRG_GROUPS_UPDATE_FILES, "create trigger update_groups_update_file" }, @@ -4783,6 +5001,7 @@ static const struct db_init_query db_init_table_queries[] = { Q_PL4, "create default smart playlist 'TV Shows'" }, { Q_PL5, "create default smart playlist 'Podcasts'" }, { Q_PL6, "create default smart playlist 'Audiobooks'" }, + { Q_DIR1, "create default root directory '/'" }, { Q_SCVER_MAJOR, "set schema version major" }, { Q_SCVER_MINOR, "set schema version minor" }, @@ -4832,12 +5051,18 @@ static const struct db_init_query db_init_table_queries[] = #define I_FILELIST \ "CREATE INDEX IF NOT EXISTS idx_filelist ON files(disabled, virtual_path, time_modified);" +#define I_FILE_DIR \ + "CREATE INDEX IF NOT EXISTS idx_file_dir ON files(disabled, directory_id);" + #define I_PL_PATH \ "CREATE INDEX IF NOT EXISTS idx_pl_path ON playlists(path);" #define I_PL_DISABLED \ "CREATE INDEX IF NOT EXISTS idx_pl_disabled ON playlists(disabled, type, virtual_path, db_timestamp);" +#define I_PL_DIR \ + "CREATE INDEX IF NOT EXISTS idx_pl_dir ON files(disabled, directory_id);" + #define I_FILEPATH \ "CREATE INDEX IF NOT EXISTS idx_filepath ON playlistitems(filepath ASC);" @@ -4850,6 +5075,12 @@ static const struct db_init_query db_init_table_queries[] = #define I_PAIRING \ "CREATE INDEX IF NOT EXISTS idx_pairingguid ON pairings(guid);" +#define I_DIR_VPATH \ + "CREATE INDEX IF NOT EXISTS idx_dir_vpath ON directories(disabled, virtual_path);" + +#define I_DIR_PARENT \ + "CREATE INDEX IF NOT EXISTS idx_dir_parentid ON directories(parent_id);" + static const struct db_init_query db_init_index_queries[] = { { I_RESCAN, "create rescan index" }, @@ -4865,9 +5096,11 @@ static const struct db_init_query db_init_index_queries[] = { I_TITLE, "create title index" }, { I_ALBUM, "create album index" }, { I_FILELIST, "create filelist index" }, + { I_FILE_DIR, "create file dir index" }, { I_PL_PATH, "create playlist path index" }, { I_PL_DISABLED, "create playlist state index" }, + { I_PL_DIR, "create playlist dir index" }, { I_FILEPATH, "create file path index" }, { I_PLITEMID, "create playlist id index" }, @@ -4875,6 +5108,9 @@ static const struct db_init_query db_init_index_queries[] = { I_GRP_PERSIST, "create groups persistentid index" }, { I_PAIRING, "create pairing guid index" }, + + { I_DIR_VPATH, "create directories disabled_virtualpath index" }, + { I_DIR_PARENT, "create directories parentid index" }, }; static int diff --git a/src/db.h b/src/db.h index fe6f7ad0..3c3ef7eb 100644 --- a/src/db.h +++ b/src/db.h @@ -61,12 +61,6 @@ enum query_type { #define ARTWORK_SPOTIFY 6 #define ARTWORK_HTTP 7 -enum filelistitem_type { - F_PLAYLIST = 1, - F_DIR = 2, - F_FILE = 3, -}; - struct query_params { /* Query parameters, filled in by caller */ enum query_type type; @@ -188,6 +182,8 @@ struct media_file_info { char *album_artist_sort; char *virtual_path; + + uint32_t directory_id; /* Id of directory */ }; #define mfi_offsetof(field) offsetof(struct media_file_info, field) @@ -215,6 +211,7 @@ struct playlist_info { uint32_t special_id; /* iTunes identifies certain 'special' playlists with special meaning */ char *virtual_path; /* virtual path of underlying playlist */ uint32_t parent_id; /* Id of parent playlist if the playlist is nested */ + uint32_t directory_id; /* Id of directory */ }; #define pli_offsetof(field) offsetof(struct playlist_info, field) @@ -233,6 +230,7 @@ struct db_playlist_info { char *special_id; char *virtual_path; char *parent_id; + char *directory_id; }; #define dbpli_offsetof(field) offsetof(struct db_playlist_info, field) @@ -322,16 +320,11 @@ struct db_media_file_info { char *composer_sort; char *album_artist_sort; char *virtual_path; + char *directory_id; }; #define dbmfi_offsetof(field) offsetof(struct db_media_file_info, field) -struct filelist_info { - char *virtual_path; - uint32_t time_modified; - enum filelistitem_type type; -}; - struct watch_info { int wd; char *path; @@ -353,15 +346,27 @@ struct filecount_info { uint32_t length; }; +struct directory_info { + uint32_t id; + char *virtual_path; + uint32_t db_timestamp; + uint32_t disabled; + uint32_t parent_id; +}; + +struct directory_enum { + int parent_id; + + /* Private enum context, keep out */ + sqlite3_stmt *stmt; +}; + char * db_escape_string(const char *str); void free_pi(struct pairing_info *pi, int content_only); -void -free_fi(struct filelist_info *fi, int content_only); - void free_mfi(struct media_file_info *mfi, int content_only); @@ -371,6 +376,9 @@ unicode_fixup_mfi(struct media_file_info *mfi); void free_pli(struct playlist_info *pli, int content_only); +void +free_di(struct directory_info *di, int content_only); + /* Maintenance and DB hygiene */ void db_hook_post_scan(void); @@ -495,6 +503,9 @@ db_file_disable_bymatch(char *path, char *strip, uint32_t cookie); int db_file_enable_bycookie(uint32_t cookie, char *path); +int +db_file_update_directoryid(char *path, int dir_id); + /* Playlists */ int db_pl_get_count(void); @@ -551,12 +562,31 @@ db_groups_clear(void); int db_group_persistentid_byid(int id, int64_t *persistentid); -/* Filelist */ + +/* Directories */ int -db_mpd_start_query_filelist(struct query_params *qp, char *path); +db_directory_id_byvirtualpath(char *virtual_path); int -db_mpd_query_fetch_filelist(struct query_params *qp, struct filelist_info *fi); +db_directory_enum_start(struct directory_enum *de); + +int +db_directory_enum_fetch(struct directory_enum *de, struct directory_info *di); + +void +db_directory_enum_end(struct directory_enum *de); + +int +db_directory_addorupdate(char *virtual_path, int disabled, int parent_id); + +void +db_directory_ping_bymatch(char *path); + +void +db_directory_disable_bymatch(char *path, char *strip, uint32_t cookie); + +int +db_directory_enable_bycookie(uint32_t cookie, char *path); /* Remotes */ int diff --git a/src/db_upgrade.c b/src/db_upgrade.c index d28ca76d..09e29f04 100644 --- a/src/db_upgrade.c +++ b/src/db_upgrade.c @@ -1167,6 +1167,263 @@ static const struct db_upgrade_query db_upgrade_v1801_queries[] = { U_V1801_SCVER_MINOR, "set schema_version_minor to 01" }, }; +/* Upgrade from schema v18.01 to v19.00 */ +/* Replace 'filelist' view with new table 'directories' + */ + +#define U_V1900_CREATE_TABLE_DIRECTORIES \ + "CREATE TABLE IF NOT EXISTS directories (" \ + " id INTEGER PRIMARY KEY NOT NULL," \ + " virtual_path VARCHAR(4096) NOT NULL," \ + " db_timestamp INTEGER DEFAULT 0," \ + " disabled INTEGER DEFAULT 0," \ + " parent_id INTEGER DEFAULT 0" \ + ");" + +#define U_V1900_DROP_VIEW_FILELIST \ + "DROP VIEW IF EXISTS filelist;" +#define U_V1900_ALTER_PL_ADD_DIRECTORYID \ + "ALTER TABLE playlists ADD COLUMN directory_id INTEGER DEFAULT 0;" +#define U_V1900_ALTER_FILES_ADD_DIRECTORYID \ + "ALTER TABLE files ADD COLUMN directory_id INTEGER DEFAULT 0;" + +#define U_V1900_INSERT_DIR1 \ + "INSERT INTO directories (id, virtual_path, db_timestamp, disabled, parent_id)" \ + " VALUES (1, '/', 0, 0, 0);" + +#define U_V1900_SCVER_MAJOR \ + "UPDATE admin SET value = '19' WHERE key = 'schema_version_major';" +#define U_V1900_SCVER_MINOR \ + "UPDATE admin SET value = '00' WHERE key = 'schema_version_minor';" + +static const struct db_upgrade_query db_upgrade_v1900_queries[] = + { + { U_V1900_CREATE_TABLE_DIRECTORIES, "create table directories" }, + { U_V1900_ALTER_PL_ADD_DIRECTORYID, "alter table pl add column directory_id" }, + { U_V1900_ALTER_FILES_ADD_DIRECTORYID, "alter table files add column directory_id" }, + { U_V1900_INSERT_DIR1, "insert root directory" }, + { U_V1900_DROP_VIEW_FILELIST, "drop view directories" }, + + { U_V1900_SCVER_MAJOR, "set schema_version_major to 19" }, + { U_V1900_SCVER_MINOR, "set schema_version_minor to 00" }, + }; + +int +db_upgrade_v19_directory_id(sqlite3 *hdl, char *virtual_path) +{ + sqlite3_stmt *stmt; + char *query; + int id; + int ret; + + query = sqlite3_mprintf("SELECT d.id FROM directories d WHERE d.disabled = 0 AND d.virtual_path = '%q';", virtual_path); + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + + return -1; + } + + ret = sqlite3_prepare_v2(hdl, query, -1, &stmt, NULL); + if (ret < 0) + { + DPRINTF(E_LOG, L_DB, "Error preparing query '%s'\n", query); + sqlite3_free(query); + + return -1; + } + + ret = sqlite3_step(stmt); + + if (ret == SQLITE_ROW) + id = sqlite3_column_int(stmt, 0); + else if (ret == SQLITE_DONE) + id = 0; // Not found + else + { + DPRINTF(E_LOG, L_DB, "Error stepping query '%s'\n", query); + sqlite3_free(query); + sqlite3_finalize(stmt); + + return -1; + } + + sqlite3_free(query); + sqlite3_finalize(stmt); + + return id; +} + +int +db_upgrade_v19_insert_directory(sqlite3 *hdl, char *virtual_path, int parent_id) +{ + char *query; + char *errmsg; + int id; + int ret; + + query = sqlite3_mprintf( + "INSERT INTO directories (virtual_path, db_timestamp, disabled, parent_id) VALUES (TRIM(%Q), %d, %d, %d);", + virtual_path, (uint64_t)time(NULL), 0, parent_id); + + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + return -1; + } + + ret = sqlite3_exec(hdl, query, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Query error: %s\n", errmsg); + + sqlite3_free(errmsg); + sqlite3_free(query); + return -1; + } + + sqlite3_free(query); + + id = (int)sqlite3_last_insert_rowid(hdl); + + DPRINTF(E_DBG, L_DB, "Added directory %s with id %d\n", virtual_path, id); + + return id; +} + +static int +db_upgrade_v19_insert_parent_directories(sqlite3 *hdl, char *virtual_path) +{ + char *ptr; + int dir_id; + int parent_id; + char buf[PATH_MAX]; + + // The root directoy ID + parent_id = 1; + + ptr = virtual_path + 1; // Skip first '/' + while (ptr && (ptr = strchr(ptr, '/'))) + { + strncpy(buf, virtual_path, (ptr - virtual_path)); + buf[(ptr - virtual_path)] = '\0'; + + dir_id = db_upgrade_v19_directory_id(hdl, buf); + + if (dir_id < 0) + { + DPRINTF(E_LOG, L_SCAN, "Select of directory failed '%s'\n", buf); + + return -1; + } + else if (dir_id == 0) + { + dir_id = db_upgrade_v19_insert_directory(hdl, buf, parent_id); + if (dir_id < 0) + { + DPRINTF(E_LOG, L_SCAN, "Insert of directory failed '%s'\n", buf); + + return -1; + } + } + + parent_id = dir_id; + ptr++; + } + + return parent_id; +} + +static int +db_upgrade_v19(sqlite3 *hdl) +{ + sqlite3_stmt *stmt; + char *query; + char *uquery; + char *errmsg; + int id; + char *virtual_path; + int dir_id; + int ret; + + query = "SELECT id, virtual_path FROM files;"; + + DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); + + ret = sqlite3_prepare_v2(hdl, query, -1, &stmt, NULL); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); + return -1; + } + + while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) + { + id = sqlite3_column_int(stmt, 0); + virtual_path = (char *)sqlite3_column_text(stmt, 1); + + dir_id = db_upgrade_v19_insert_parent_directories(hdl, virtual_path); + if (dir_id < 0) + { + DPRINTF(E_LOG, L_DB, "Error processing parent directories for file: %s\n", virtual_path); + } + else + { + uquery = sqlite3_mprintf("UPDATE files SET directory_id = %d WHERE id = %d;", dir_id, id); + ret = sqlite3_exec(hdl, uquery, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Error updating files: %s\n", errmsg); + } + + sqlite3_free(uquery); + sqlite3_free(errmsg); + } + } + + sqlite3_finalize(stmt); + + + query = "SELECT id, virtual_path FROM playlists WHERE type = 2 OR type = 3;"; //Only update normal and smart playlists + + DPRINTF(E_DBG, L_DB, "Running query '%s'\n", query); + + ret = sqlite3_prepare_v2(hdl, query, -1, &stmt, NULL); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Could not prepare statement: %s\n", sqlite3_errmsg(hdl)); + return -1; + } + + while ((ret = sqlite3_step(stmt)) == SQLITE_ROW) + { + id = sqlite3_column_int(stmt, 0); + virtual_path = (char *)sqlite3_column_text(stmt, 1); + + dir_id = db_upgrade_v19_insert_parent_directories(hdl, virtual_path); + if (dir_id < 0) + { + DPRINTF(E_LOG, L_DB, "Error processing parent directories for file: %s\n", virtual_path); + } + else + { + uquery = sqlite3_mprintf("UPDATE files SET directory_id = %d WHERE id = %d;", dir_id, id); + ret = sqlite3_exec(hdl, uquery, NULL, NULL, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Error updating files: %s\n", errmsg); + } + + sqlite3_free(uquery); + sqlite3_free(errmsg); + } + } + + sqlite3_finalize(stmt); + + return 0; +} + int db_upgrade(sqlite3 *hdl, int db_ver) { @@ -1266,6 +1523,17 @@ db_upgrade(sqlite3 *hdl, int db_ver) if (ret < 0) return -1; + /* FALLTHROUGH */ + + case 1801: + ret = db_generic_upgrade(hdl, db_upgrade_v1900_queries, sizeof(db_upgrade_v1900_queries) / sizeof(db_upgrade_v1900_queries[0])); + if (ret < 0) + return -1; + + ret = db_upgrade_v19(hdl); + if (ret < 0) + return -1; + break; default: diff --git a/src/filescanner.c b/src/filescanner.c index 7e500ac7..7d5862e6 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -112,13 +112,20 @@ struct deferred_pl { char *path; time_t mtime; struct deferred_pl *next; + int directory_id; }; struct stacked_dir { char *path; + int parent_id; struct stacked_dir *next; }; +enum root_directories { + DIR_FILE = 0, + DIR_HTTP = 1, + DIR_SPOTIFY = 2, +}; static int cmd_pipe[2]; static int exit_pipe[2]; @@ -197,7 +204,7 @@ nonblock_command(struct filescanner_command *cmd) } static int -push_dir(struct stacked_dir **s, char *path) +push_dir(struct stacked_dir **s, char *path, int parent_id) { struct stacked_dir *d; @@ -215,28 +222,26 @@ push_dir(struct stacked_dir **s, char *path) return -1; } + d->parent_id = parent_id; + d->next = *s; *s = d; return 0; } -static char * +static struct stacked_dir * pop_dir(struct stacked_dir **s) { struct stacked_dir *d; - char *ret; if (!*s) return NULL; d = *s; *s = d->next; - ret = d->path; - free(d); - - return ret; + return d; } #ifdef HAVE_REGEX_H @@ -633,7 +638,7 @@ fixup_tags(struct media_file_info *mfi) void -filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi) +filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi, int dir_id) { struct media_file_info *mfi; char *filename; @@ -765,6 +770,8 @@ filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct mfi->virtual_path = strdup(virtual_path); } + mfi->directory_id = dir_id; + if (mfi->id == 0) db_file_add(mfi); else @@ -776,13 +783,13 @@ filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct } static void -process_playlist(char *file, time_t mtime) +process_playlist(char *file, time_t mtime, int dir_id) { enum file_type ft; ft = file_type_get(file); if (ft == FILE_PLAYLIST) - scan_playlist(file, mtime); + scan_playlist(file, mtime, dir_id); #ifdef ITUNES else if (ft == FILE_ITUNES) scan_itunes_itml(file); @@ -791,7 +798,7 @@ process_playlist(char *file, time_t mtime) /* Thread: scan */ static void -defer_playlist(char *path, time_t mtime) +defer_playlist(char *path, time_t mtime, int dir_id) { struct deferred_pl *pl; @@ -815,6 +822,7 @@ defer_playlist(char *path, time_t mtime) } pl->mtime = mtime; + pl->directory_id = dir_id; pl->next = playlists; playlists = pl; @@ -831,7 +839,7 @@ process_deferred_playlists(void) { playlists = pl->next; - process_playlist(pl->path, pl->mtime); + process_playlist(pl->path, pl->mtime, pl->directory_id); free(pl->path); free(pl); @@ -843,7 +851,7 @@ process_deferred_playlists(void) /* Thread: scan */ static void -process_file(char *file, time_t mtime, off_t size, int type, int flags) +process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_id) { int is_bulkscan; @@ -852,7 +860,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags) switch (file_type_get(file)) { case FILE_REGULAR: - filescanner_process_media(file, mtime, size, type, NULL); + filescanner_process_media(file, mtime, size, type, NULL, dir_id); cache_artwork_ping(file, mtime, !is_bulkscan); // TODO [artworkcache] If entry in artwork cache exists for no artwork available, delete the entry if media file has embedded artwork @@ -871,14 +879,14 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags) case FILE_PLAYLIST: case FILE_ITUNES: if (flags & F_SCAN_BULK) - defer_playlist(file, mtime); + defer_playlist(file, mtime, dir_id); else - process_playlist(file, mtime); + process_playlist(file, mtime, dir_id); break; case FILE_SMARTPL: DPRINTF(E_DBG, L_SCAN, "Smart playlist file: %s\n", file); - scan_smartpl(file, mtime); + scan_smartpl(file, mtime, dir_id); break; case FILE_ARTWORK: @@ -948,8 +956,22 @@ check_speciallib(char *path, const char *libtype) } /* Thread: scan */ +static int +create_virtual_path(char *path, char *virtual_path, int virtual_path_len) +{ + int ret; + ret = snprintf(virtual_path, virtual_path_len, "/file:%s", path); + if ((ret < 0) || (ret >= virtual_path_len)) + { + DPRINTF(E_LOG, L_SCAN, "Virtual path /file:%s, PATH_MAX exceeded\n", path); + return -1; + } + + return 0; +} + static void -process_directory(char *path, int flags) +process_directory(char *path, int flags, int parent_id) { DIR *dirp; struct dirent buf; @@ -962,6 +984,9 @@ process_directory(char *path, int flags) struct kevent kev; #endif int type; + struct directory_info di; + char virtual_path[PATH_MAX]; + int dir_id; int ret; DPRINTF(E_DBG, L_SCAN, "Processing directory %s (flags = 0x%x)\n", path, flags); @@ -974,6 +999,24 @@ process_directory(char *path, int flags) return; } + /* Add/update directories table */ + memset(&di, 0, sizeof(struct directory_info)); + + ret = create_virtual_path(path, virtual_path, sizeof(virtual_path)); + if (ret < 0) + { + DPRINTF(E_LOG, L_SCAN, "Virtual path /file:%s, PATH_MAX exceeded\n", path); + + return; + } + + dir_id = db_directory_addorupdate(virtual_path, 0, parent_id); + + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '/http:'\n"); + } + /* Check if compilation and/or podcast directory */ type = 0; if (check_speciallib(path, "compilations")) @@ -1050,15 +1093,15 @@ process_directory(char *path, int flags) if (S_ISREG(sb.st_mode)) { if (!(flags & F_SCAN_FAST)) - process_file(entry, sb.st_mtime, sb.st_size, F_SCAN_TYPE_FILE | type, flags); + process_file(entry, sb.st_mtime, sb.st_size, F_SCAN_TYPE_FILE | type, flags, dir_id); } else if (S_ISFIFO(sb.st_mode)) { if (!(flags & F_SCAN_FAST)) - process_file(entry, sb.st_mtime, sb.st_size, F_SCAN_TYPE_PIPE | type, flags); + process_file(entry, sb.st_mtime, sb.st_size, F_SCAN_TYPE_PIPE | type, flags, dir_id); } else if (S_ISDIR(sb.st_mode)) - push_dir(&dirstack, entry); + push_dir(&dirstack, entry, dir_id); else DPRINTF(E_LOG, L_SCAN, "Skipping %s, not a directory, symlink, pipe nor regular file\n", entry); } @@ -1116,21 +1159,63 @@ process_directory(char *path, int flags) } /* Thread: scan */ -static void -process_directories(char *root, int flags) -{ - char *path; - process_directory(root, flags); +static int +process_parent_directories(char *path) +{ + char *ptr; + int dir_id; + char buf[PATH_MAX]; + char virtual_path[PATH_MAX]; + int ret; + + // The root directoy ID + dir_id = 1; + + ptr = path; + while (ptr && (ptr = strchr(ptr, '/'))) + { + strncpy(buf, path, (ptr - path)); + buf[(ptr - path)] = '\0'; + + ret = create_virtual_path(buf, virtual_path, sizeof(virtual_path)); + if (ret < 0) + { + DPRINTF(E_LOG, L_SCAN, "Virtual path /file:%s, PATH_MAX exceeded\n", path); + return 0; + } + + dir_id = db_directory_addorupdate(virtual_path, 0, dir_id); + + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '%s'\n", virtual_path); + + return 0; + } + + ptr++; + } + + return dir_id; +} + +static void +process_directories(char *root, int parent_id, int flags) +{ + struct stacked_dir *dir; + + process_directory(root, flags, parent_id); if (scan_exit) return; - while ((path = pop_dir(&dirstack))) + while ((dir = pop_dir(&dirstack))) { - process_directory(path, flags); + process_directory(dir->path, flags, dir->parent_id); - free(path); + free(dir->path); + free(dir); if (scan_exit) return; @@ -1148,6 +1233,7 @@ bulk_scan(int flags) char *deref; time_t start; time_t end; + int parent_id; int i; // Set global flag to avoid queued scan requests @@ -1165,6 +1251,8 @@ bulk_scan(int flags) { path = cfg_getnstr(lib, "directories", i); + parent_id = process_parent_directories(path); + deref = m_realpath(path); if (!deref) { @@ -1173,16 +1261,19 @@ bulk_scan(int flags) /* Assume dir is mistakenly not mounted, so just disable everything and update timestamps */ db_file_disable_bymatch(path, "", 0); db_pl_disable_bymatch(path, "", 0); + db_directory_disable_bymatch(path, "", 0); db_file_ping_bymatch(path, 1); db_pl_ping_bymatch(path, 1); + db_directory_ping_bymatch(path); continue; } counter = 0; db_transaction_begin(); - process_directories(deref, flags); + + process_directories(deref, parent_id, flags); db_transaction_end(); free(deref); @@ -1347,6 +1438,7 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) char *s; int flags = 0; int ret; + int parent_id; DPRINTF(E_SPAM, L_SCAN, "Directory event: 0x%x, cookie 0x%x, wd %d\n", ie->mask, ie->cookie, wi->wd); @@ -1354,6 +1446,7 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) { db_file_disable_bymatch(path, "", 0); db_pl_disable_bymatch(path, "", 0); + db_directory_disable_bymatch(path, "", 0); } if (ie->mask & IN_MOVE_SELF) @@ -1408,6 +1501,7 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) db_watch_mark_bymatch(path, path, ie->cookie); db_file_disable_bymatch(path, path, ie->cookie); db_pl_disable_bymatch(path, path, ie->cookie); + db_directory_disable_bymatch(path, path, ie->cookie); } if (ie->mask & IN_MOVED_TO) @@ -1417,6 +1511,7 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) db_watch_move_bycookie(ie->cookie, path); db_file_enable_bycookie(ie->cookie, path); db_pl_enable_bycookie(ie->cookie, path); + db_directory_enable_bycookie(ie->cookie, path); /* We'll rescan the directory tree to update playlists */ flags |= F_SCAN_MOVED; @@ -1450,6 +1545,7 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) db_file_disable_bymatch(path, "", 0); db_pl_disable_bymatch(path, "", 0); + db_directory_disable_bymatch(path, "", 0); } else if (ret < 0) { @@ -1465,7 +1561,9 @@ process_inotify_dir(struct watch_info *wi, char *path, struct inotify_event *ie) if (ie->mask & IN_CREATE) { - process_directories(path, flags); + parent_id = process_parent_directories(path); + + process_directories(path, parent_id, flags); if (dirstack) DPRINTF(E_LOG, L_SCAN, "WARNING: unhandled leftover directories\n"); @@ -1480,8 +1578,12 @@ process_inotify_file(struct watch_info *wi, char *path, struct inotify_event *ie uint32_t path_hash; char *deref = NULL; char *file = path; + char *dir; + char dir_vpath[PATH_MAX]; int type; int i; + int dir_id; + char *ptr; int ret; DPRINTF(E_SPAM, L_SCAN, "File event: 0x%x, cookie 0x%x, wd %d\n", ie->mask, ie->cookie, wi->wd); @@ -1541,7 +1643,30 @@ process_inotify_file(struct watch_info *wi, char *path, struct inotify_event *ie ret = db_file_enable_bycookie(ie->cookie, path); - if (ret <= 0) + if (ret > 0) + { + // If file was successfully enabled, update the directory id + dir = strdup(path); + ptr = strrchr(dir, '/'); + dir[(ptr - dir)] = '\0'; + + ret = create_virtual_path(dir, dir_vpath, sizeof(dir_vpath)); + if (ret < 0) + DPRINTF(E_LOG, L_SCAN, "Error creating virtual path for: %s\n", dir); + else + { + dir_id = db_directory_id_byvirtualpath(dir_vpath); + if (dir_id > 0) + { + ret = db_file_update_directoryid(path, dir_id); + if (ret < 0) + DPRINTF(E_LOG, L_SCAN, "Error updating directory id for file: %s\n", path); + } + } + + free(dir); + } + else { /* It's not a known media file, so it's either a new file * or a playlist, known or not. @@ -1636,10 +1761,14 @@ process_inotify_file(struct watch_info *wi, char *path, struct inotify_event *ie if (check_speciallib(path, "audiobooks")) type |= F_SCAN_TYPE_AUDIOBOOK; + dir_id = process_parent_directories(file); + if (S_ISREG(sb.st_mode)) - process_file(file, sb.st_mtime, sb.st_size, F_SCAN_TYPE_FILE | type, 0); + { + process_file(file, sb.st_mtime, sb.st_size, F_SCAN_TYPE_FILE | type, 0, dir_id); + } else if (S_ISFIFO(sb.st_mode)) - process_file(file, sb.st_mtime, sb.st_size, F_SCAN_TYPE_PIPE | type, 0); + process_file(file, sb.st_mtime, sb.st_size, F_SCAN_TYPE_PIPE | type, 0, dir_id); if (deref) free(deref); diff --git a/src/filescanner.h b/src/filescanner.h index f9136a86..915f99b0 100644 --- a/src/filescanner.h +++ b/src/filescanner.h @@ -19,7 +19,7 @@ void filescanner_deinit(void); void -filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi); +filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi, int dir_id); /* Actual scanners */ int @@ -29,10 +29,10 @@ int scan_metadata_icy(char *url, struct media_file_info *mfi); void -scan_playlist(char *file, time_t mtime); +scan_playlist(char *file, time_t mtime, int dir_id); void -scan_smartpl(char *file, time_t mtime); +scan_smartpl(char *file, time_t mtime, int dir_id); #ifdef ITUNES void diff --git a/src/filescanner_playlist.c b/src/filescanner_playlist.c index 605cc3c6..8a7444ea 100644 --- a/src/filescanner_playlist.c +++ b/src/filescanner_playlist.c @@ -74,7 +74,7 @@ extinf_get(char *string, struct media_file_info *mfi, int *extinf) } void -scan_playlist(char *file, time_t mtime) +scan_playlist(char *file, time_t mtime, int dir_id) { FILE *fp; struct media_file_info mfi; @@ -93,6 +93,7 @@ scan_playlist(char *file, time_t mtime) int ret; char virtual_path[PATH_MAX]; int i; + int di_id; DPRINTF(E_LOG, L_SCAN, "Processing static playlist: %s\n", file); @@ -172,6 +173,8 @@ scan_playlist(char *file, time_t mtime) *ptr = '\0'; pli->virtual_path = strdup(virtual_path); + pli->directory_id = dir_id; + ret = db_pl_add(pli, &pl_id); if (ret < 0) { @@ -236,7 +239,14 @@ scan_playlist(char *file, time_t mtime) if (extinf) DPRINTF(E_INFO, L_SCAN, "Playlist has EXTINF metadata, artist is '%s', title is '%s'\n", mfi.artist, mfi.title); - filescanner_process_media(filename, mtime, 0, F_SCAN_TYPE_URL, &mfi); + di_id = db_directory_addorupdate("/http:", 0, 1); + + if (di_id <= 0) + { + DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '/http:'\n"); + } + + filescanner_process_media(filename, mtime, 0, F_SCAN_TYPE_URL, &mfi, di_id); } /* Regular file, should already be in library */ else diff --git a/src/filescanner_smartpl.c b/src/filescanner_smartpl.c index 53bb60eb..5c55ef4e 100644 --- a/src/filescanner_smartpl.c +++ b/src/filescanner_smartpl.c @@ -171,7 +171,7 @@ smartpl_parse_file(const char *file, struct playlist_info *pli) } void -scan_smartpl(char *file, time_t mtime) +scan_smartpl(char *file, time_t mtime, int dir_id) { struct playlist_info *pli; int pl_id; @@ -203,6 +203,8 @@ scan_smartpl(char *file, time_t mtime) else pl_id = pli->id; + pli->directory_id = dir_id; + ret = smartpl_parse_file(file, pli); if (ret < 0) { diff --git a/src/mpd.c b/src/mpd.c index 0be72991..34a10319 100644 --- a/src/mpd.c +++ b/src/mpd.c @@ -2692,12 +2692,16 @@ mpd_command_list(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) static int mpd_command_lsinfo(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) { + int dir_id; + struct directory_info subdir; struct query_params qp; + struct directory_enum dir_enum; char parent[PATH_MAX]; - struct filelist_info *fi; - struct media_file_info *mfi; - char modified[32]; int print_playlists; + struct db_playlist_info dbpli; + char modified[32]; + uint32_t time_modified; + struct db_media_file_info dbmfi; int ret; if (argc < 2 || strlen(argv[1]) == 0 @@ -2711,7 +2715,7 @@ mpd_command_lsinfo(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) } else { - ret = snprintf(parent, sizeof(parent), "/%s/", argv[1]); + ret = snprintf(parent, sizeof(parent), "/%s", argv[1]); } if ((ret < 0) || (ret >= sizeof(parent))) @@ -2731,72 +2735,94 @@ mpd_command_lsinfo(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) print_playlists = 1; } - fi = (struct filelist_info*)malloc(sizeof(struct filelist_info)); - if (!fi) + + // Load dir-id from db for parent-path + dir_id = db_directory_id_byvirtualpath(parent); + if (dir_id == 0) { - DPRINTF(E_LOG, L_MPD, "Out of memory for fi\n"); - return ACK_ERROR_UNKNOWN; + DPRINTF(E_LOG, L_MPD, "Directory info not found for virtual-path '%s'\n", parent); + return -1; } + // Load playlists for dir-id memset(&qp, 0, sizeof(struct query_params)); - - ret = db_mpd_start_query_filelist(&qp, parent); + qp.type = Q_PL; + qp.sort = S_PLAYLIST; + qp.idx_type = I_NONE; + qp.filter = sqlite3_mprintf("(directory_id = %d AND (f.type = %d OR f.type = %d))", dir_id, PL_PLAIN, PL_SMART); + ret = db_query_start(&qp); if (ret < 0) { - ret = asprintf(errmsg, "Could not start query for path '%s'", argv[1]); + db_query_end(&qp); + ret = asprintf(errmsg, "Could not start query"); if (ret < 0) - DPRINTF(E_LOG, L_MPD, "Out of memory\n"); - - free_fi(fi, 0); + DPRINTF(E_LOG, L_MPD, "Out of memory\n"); return ACK_ERROR_UNKNOWN; } - - while (((ret = db_mpd_query_fetch_filelist(&qp, fi)) == 0) && (fi->virtual_path)) + while (((ret = db_query_fetch_pl(&qp, &dbpli)) == 0) && (dbpli.id)) { - if (fi->type == F_DIR) + if (safe_atou32(dbpli.db_timestamp, &time_modified) != 0) { - mpd_time(modified, sizeof(modified), fi->time_modified); - - evbuffer_add_printf(evbuf, - "directory: %s\n" - "Last-Modified: %s\n", - (fi->virtual_path + 1), - modified); - } - else if (fi->type == F_PLAYLIST) - { - mpd_time(modified, sizeof(modified), fi->time_modified); - - evbuffer_add_printf(evbuf, - "playlist: %s\n" - "Last-Modified: %s\n", - (fi->virtual_path + 1), - modified); - } - else if (fi->type == F_FILE) - { - mfi = db_file_fetch_byvirtualpath(fi->virtual_path); - if (mfi) - { - ret = mpd_add_mediainfo(evbuf, mfi, 0, -1); - if (ret < 0) - { - DPRINTF(E_LOG, L_MPD, "Could not add mediainfo for path '%s'\n", fi->virtual_path); - } - - free_mfi(mfi, 0); - } + DPRINTF(E_LOG, L_MPD, "Error converting time modified to uint32_t: %s\n", dbpli.db_timestamp); + return -1; } + mpd_time(modified, sizeof(modified), time_modified); + evbuffer_add_printf(evbuf, + "playlist: %s\n" + "Last-Modified: %s\n", + (dbpli.virtual_path + 1), + modified); } + db_query_end(&qp); + sqlite3_free(qp.filter); + // Load sub directories for dir-id + memset(&dir_enum, 0, sizeof(struct directory_enum)); + dir_enum.parent_id = dir_id; + ret = db_directory_enum_start(&dir_enum); + if (ret < 0) + { + DPRINTF(E_LOG, L_MPD, "Failed to start directory enum for parent_id %d\n", dir_id); + return -1; + } + while ((ret = db_directory_enum_fetch(&dir_enum, &subdir)) == 0 && subdir.id > 0) + { + evbuffer_add_printf(evbuf, + "directory: %s\n" + "Last-Modified: %s\n", + (subdir.virtual_path + 1), + "2015-12-01 00:00"); + } + db_directory_enum_end(&dir_enum); + + // Load files for dir-id + memset(&qp, 0, sizeof(struct query_params)); + qp.type = Q_ITEMS; + qp.sort = S_ARTIST; + qp.idx_type = I_NONE; + qp.filter = sqlite3_mprintf("(directory_id = %d)", dir_id); + ret = db_query_start(&qp); + if (ret < 0) + { + db_query_end(&qp); + ret = asprintf(errmsg, "Could not start query"); + if (ret < 0) + DPRINTF(E_LOG, L_MPD, "Out of memory\n"); + return ACK_ERROR_UNKNOWN; + } + while (((ret = db_query_fetch_file(&qp, &dbmfi)) == 0) && (dbmfi.id)) + { + ret = mpd_add_db_media_file_info(evbuf, &dbmfi); + if (ret < 0) + { + DPRINTF(E_LOG, L_MPD, "Error adding song to the evbuffer, song id: %s\n", dbmfi.id); + } + } db_query_end(&qp); - if (fi) - free_fi(fi, 0); - + // If the root directory was passed as argument add the stored playlists to the response if (print_playlists) { - // If the root directory was passed as argument add the stored playlists to the response return mpd_command_listplaylists(evbuf, argc, argv, errmsg); } diff --git a/src/spotify.c b/src/spotify.c index 5e75b6b3..1f29d56c 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -573,6 +574,9 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde sp_link *link; char url[1024]; int ret; + int dir_id; + char virtual_path[PATH_MAX]; + if (!fptr_sp_track_is_loaded(track)) { @@ -618,7 +622,32 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde return -1; } - filescanner_process_media(url, time(NULL), 0, F_SCAN_TYPE_SPOTIFY, &mfi); + snprintf(virtual_path, PATH_MAX, "/spotify:"); + dir_id = db_directory_addorupdate(virtual_path, 0, 1); + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + free_mfi(&mfi, 1); + return -1; + } + snprintf(virtual_path, PATH_MAX, "/spotify:/%s", mfi.artist); + dir_id = db_directory_addorupdate(virtual_path, 0, dir_id); + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + free_mfi(&mfi, 1); + return -1; + } + snprintf(virtual_path, PATH_MAX, "/spotify:/%s/%s", mfi.artist, mfi.album); + dir_id = db_directory_addorupdate(virtual_path, 0, dir_id); + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + free_mfi(&mfi, 1); + return -1; + } + + filescanner_process_media(url, time(NULL), 0, F_SCAN_TYPE_SPOTIFY, &mfi, dir_id); free_mfi(&mfi, 1); @@ -669,9 +698,10 @@ spotify_playlist_save(sp_playlist *pl) int plid; int num_tracks; char virtual_path[PATH_MAX]; - int time; + int created; int ret; int i; + int dir_id; if (!fptr_sp_playlist_is_loaded(pl)) { @@ -752,6 +782,16 @@ spotify_playlist_save(sp_playlist *pl) return -1; } + dir_id = db_directory_addorupdate("/spotify:", 0, 1); + + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '/spotify:'\n"); + + free_pli(pli, 0); + return -1; + } + memset(pli, 0, sizeof(struct playlist_info)); pli->type = PL_PLAIN; @@ -759,6 +799,7 @@ spotify_playlist_save(sp_playlist *pl) pli->path = strdup(url); pli->virtual_path = strdup(virtual_path); pli->parent_id = g_base_plid; + pli->directory_id = dir_id; ret = db_pl_add(pli, &plid); if ((ret < 0) || (plid < 1)) @@ -783,9 +824,9 @@ spotify_playlist_save(sp_playlist *pl) continue; } - time = fptr_sp_playlist_track_create_time(pl, i); + created = fptr_sp_playlist_track_create_time(pl, i); - ret = spotify_track_save(plid, track, name, time); + ret = spotify_track_save(plid, track, name, created); if (ret < 0) { DPRINTF(E_LOG, L_SPOTIFY, "Error saving track %d to playlist '%s' (id %d)\n", i, name, plid);