From 1d2e4dc7a8b2e8eb6cf55eaf4de9b13669325870 Mon Sep 17 00:00:00 2001 From: chme Date: Tue, 28 Dec 2021 09:19:44 +0100 Subject: [PATCH] [db,library] Add "scan_kind" field to playlists, directories and files (db upgrade to v22.00) `scan_kind` identifies the library "scanner" component that created the item and is responsible to keep it up to date (rescan). The library update now supports passing a `scan_kind` to update only the items of one particular "scanner". This allows e. g. to only update the item from the Spotify library or only update the RSS feeds. The OwnTone database is upgraded to v22.00 and the `scan_kind` columns in `files`, `playlists`, `directories` are identified by: 1. Check if item is part of a RSS playlist (podcast RSS feed), they belong to the "rssscanner" 2. Check if item has a Spotify `virtual_path`, they belong to the "spotifyscanner" 3. Remaining items belong to the "filescanner" --- src/db.c | 150 ++++++++++++++++++++-------- src/db.h | 30 +++++- src/db_init.c | 9 +- src/db_init.h | 4 +- src/db_upgrade.c | 60 +++++++++++ src/httpd_jsonapi.c | 29 +++++- src/library.c | 154 +++++++++++++++++++++-------- src/library.h | 26 ++++- src/library/filescanner.c | 12 ++- src/library/filescanner_playlist.c | 2 + src/library/rssscanner.c | 4 +- src/library/spotify_webapi.c | 14 ++- src/mpd.c | 2 +- 13 files changed, 390 insertions(+), 106 deletions(-) diff --git a/src/db.c b/src/db.c index 165a696f..dcc0a2d0 100644 --- a/src/db.c +++ b/src/db.c @@ -228,6 +228,7 @@ static const struct col_type_map mfi_cols_map[] = { "composer_sort", mfi_offsetof(composer_sort), DB_TYPE_STRING, DB_FIXUP_COMPOSER_SORT }, { "channels", mfi_offsetof(channels), DB_TYPE_INT }, { "usermark", mfi_offsetof(usermark), DB_TYPE_INT }, + { "scan_kind", mfi_offsetof(scan_kind), DB_TYPE_INT }, }; /* This list must be kept in sync with @@ -252,6 +253,7 @@ static const struct col_type_map pli_cols_map[] = { "query_limit", pli_offsetof(query_limit), DB_TYPE_INT }, { "media_kind", pli_offsetof(media_kind), DB_TYPE_INT, DB_FIXUP_MEDIA_KIND }, { "artwork_url", pli_offsetof(artwork_url), DB_TYPE_STRING, DB_FIXUP_NO_SANITIZE }, + { "scan_kind", pli_offsetof(scan_kind), DB_TYPE_INT }, // Not in the database, but returned via the query's COUNT()/SUM() { "items", pli_offsetof(items), DB_TYPE_INT, DB_FIXUP_STANDARD, DB_FLAG_NO_BIND }, @@ -367,6 +369,7 @@ static const ssize_t dbmfi_cols_map[] = dbmfi_offsetof(composer_sort), dbmfi_offsetof(channels), dbmfi_offsetof(usermark), + dbmfi_offsetof(scan_kind), }; /* This list must be kept in sync with @@ -391,6 +394,7 @@ static const ssize_t dbpli_cols_map[] = dbpli_offsetof(query_limit), dbpli_offsetof(media_kind), dbpli_offsetof(artwork_url), + dbpli_offsetof(scan_kind), dbpli_offsetof(items), dbpli_offsetof(streams), @@ -528,14 +532,14 @@ static const struct browse_clause browse_clause[] = }; -struct media_kind_label { - enum media_kind type; +struct enum_label { + int type; const char *label; }; /* Keep in sync with enum media_kind */ -static const struct media_kind_label media_kind_labels[] = +static const struct enum_label media_kind_labels[] = { { MEDIA_KIND_MUSIC, "music" }, { MEDIA_KIND_MOVIE, "movie" }, @@ -590,6 +594,43 @@ db_data_kind_label(enum data_kind data_kind) return NULL; } +/* Keep in sync with enum scan_kind */ +static const struct enum_label scan_kind_labels[] = + { + { SCAN_KIND_UNKNOWN, "unknown" }, + { SCAN_KIND_FILES, "files" }, + { SCAN_KIND_SPOTIFY, "spotify" }, + { SCAN_KIND_RSS, "rss" }, + }; + +const char * +db_scan_kind_label(enum scan_kind scan_kind) +{ + if (scan_kind < ARRAY_SIZE(scan_kind_labels)) + { + return scan_kind_labels[scan_kind].label; + } + + return NULL; +} + +enum scan_kind +db_scan_kind_enum(const char *name) +{ + int i; + + if (!name) + return 0; + + for (i = 0; i < ARRAY_SIZE(scan_kind_labels); i++) + { + if (strcmp(name, scan_kind_labels[i].label) == 0) + return scan_kind_labels[i].type; + } + + return 0; +} + /* Keep in sync with enum pl_type */ static char *pl_type_label[] = { "special", "folder", "smart", "plain", "rss" }; @@ -767,6 +808,7 @@ free_di(struct directory_info *di, int content_only) if (!di) return; + free(di->path); free(di->virtual_path); if (!content_only) @@ -1675,6 +1717,59 @@ db_purge_cruft(time_t ref) #undef Q_TMPL } +void +db_purge_cruft_bysource(time_t ref, enum scan_kind scan_kind) +{ +#define Q_TMPL "DELETE FROM directories WHERE id >= %d AND db_timestamp < %" PRIi64 " AND scan_kind = %d;" + int i; + int ret; + char *query; + char *queries_tmpl[4] = + { + "DELETE FROM playlistitems WHERE playlistid IN (SELECT p.id FROM playlists p WHERE p.type <> %d AND p.db_timestamp < %" PRIi64 " AND scan_kind = %d);", + "DELETE FROM playlistitems WHERE filepath IN (SELECT f.path FROM files f WHERE -1 <> %d AND f.db_timestamp < %" PRIi64 " AND scan_kind = %d);", + "DELETE FROM playlists WHERE type <> %d AND db_timestamp < %" PRIi64 " AND scan_kind = %d;", + "DELETE FROM files WHERE -1 <> %d AND db_timestamp < %" PRIi64 " AND scan_kind = %d;", + }; + + db_transaction_begin(); + + for (i = 0; i < (sizeof(queries_tmpl) / sizeof(queries_tmpl[0])); i++) + { + query = sqlite3_mprintf(queries_tmpl[i], PL_SPECIAL, (int64_t)ref, scan_kind); + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + db_transaction_end(); + return; + } + + DPRINTF(E_DBG, L_DB, "Running purge query '%s'\n", query); + + ret = db_query_run(query, 1, 0); + if (ret == 0) + DPRINTF(E_DBG, L_DB, "Purged %d rows\n", sqlite3_changes(hdl)); + } + + query = sqlite3_mprintf(Q_TMPL, DIR_MAX, (int64_t)ref, scan_kind); + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + db_transaction_end(); + return; + } + + DPRINTF(E_DBG, L_DB, "Running purge query '%s'\n", query); + + ret = db_query_run(query, 1, LISTENER_DATABASE); + if (ret == 0) + DPRINTF(E_DBG, L_DB, "Purged %d rows\n", sqlite3_changes(hdl)); + + db_transaction_end(); + +#undef Q_TMPL +} + void db_purge_all(void) { @@ -4218,6 +4313,7 @@ db_directory_enum_fetch(struct directory_enum *de, struct directory_info *di) di->disabled = sqlite3_column_int64(de->stmt, 3); di->parent_id = sqlite3_column_int(de->stmt, 4); di->path = (char *)sqlite3_column_text(de->stmt, 5); + di->scan_kind = sqlite3_column_int(de->stmt, 6); return 0; } @@ -4232,11 +4328,11 @@ db_directory_enum_end(struct directory_enum *de) de->stmt = NULL; } -static int +int db_directory_add(struct directory_info *di, int *id) { -#define QADD_TMPL "INSERT INTO directories (virtual_path, db_timestamp, disabled, parent_id, path)" \ - " VALUES (TRIM(%Q), %d, %" PRIi64 ", %d, TRIM(%Q));" +#define QADD_TMPL "INSERT INTO directories (virtual_path, db_timestamp, disabled, parent_id, path, scan_kind)" \ + " VALUES (TRIM(%Q), %d, %" PRIi64 ", %d, TRIM(%Q), %d);" char *query; char *errmsg; @@ -4251,7 +4347,7 @@ db_directory_add(struct directory_info *di, int *id) DPRINTF(E_LOG, L_DB, "Directory name ends with space: '%s'\n", di->virtual_path); } - query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id, di->path); + query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id, di->path, di->scan_kind); if (!query) { @@ -4287,17 +4383,17 @@ db_directory_add(struct directory_info *di, int *id) #undef QADD_TMPL } -static int +int db_directory_update(struct directory_info *di) { -#define QADD_TMPL "UPDATE directories SET virtual_path = TRIM(%Q), db_timestamp = %d, disabled = %" PRIi64 ", parent_id = %d, path = TRIM(%Q)" \ +#define QADD_TMPL "UPDATE directories SET virtual_path = TRIM(%Q), db_timestamp = %d, disabled = %" PRIi64 ", parent_id = %d, path = TRIM(%Q), scan_kind = %d" \ " WHERE id = %d;" char *query; char *errmsg; int ret; /* Add */ - query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id, di->path, di->id); + query = sqlite3_mprintf(QADD_TMPL, di->virtual_path, di->db_timestamp, di->disabled, di->parent_id, di->path, di->scan_kind, di->id); if (!query) { @@ -4326,36 +4422,6 @@ db_directory_update(struct directory_info *di) #undef QADD_TMPL } -int -db_directory_addorupdate(char *virtual_path, char *path, int disabled, int parent_id) -{ - struct directory_info di; - int id; - int ret; - - id = db_directory_id_byvirtualpath(virtual_path); - - di.id = id; - di.parent_id = parent_id; - di.virtual_path = virtual_path; - di.path = path; - di.disabled = disabled; - di.db_timestamp = (uint64_t)time(NULL); - - if (di.id == 0) - ret = db_directory_add(&di, &id); - else - ret = db_directory_update(&di); - - if (ret < 0 || id <= 0) - { - DPRINTF(E_LOG, L_DB, "Insert or update of directory failed '%s'\n", virtual_path); - return -1; - } - - return id; -} - void db_directory_ping_bymatch(char *virtual_path) { @@ -4369,7 +4435,7 @@ db_directory_ping_bymatch(char *virtual_path) } void -db_directory_disable_bymatch(char *path, enum strip_type strip, uint32_t cookie) +db_directory_disable_bymatch(const char *path, enum strip_type strip, uint32_t cookie) { #define Q_TMPL "UPDATE directories SET virtual_path = substr(virtual_path, %d)," \ " disabled = %" PRIi64 " WHERE virtual_path = '/file:%q' OR virtual_path LIKE '/file:%q/%%';" @@ -4388,7 +4454,7 @@ db_directory_disable_bymatch(char *path, enum strip_type strip, uint32_t cookie) } int -db_directory_enable_bycookie(uint32_t cookie, char *path) +db_directory_enable_bycookie(uint32_t cookie, const char *path) { #define Q_TMPL "UPDATE directories SET virtual_path = ('/file:%q' || virtual_path)," \ " disabled = 0 WHERE disabled = %" PRIi64 ";" diff --git a/src/db.h b/src/db.h index 8f547059..61701a85 100644 --- a/src/db.h +++ b/src/db.h @@ -141,6 +141,18 @@ enum data_kind { const char * db_data_kind_label(enum data_kind data_kind); +enum scan_kind { + SCAN_KIND_UNKNOWN = 0, + SCAN_KIND_FILES = 1, + SCAN_KIND_SPOTIFY = 2, + SCAN_KIND_RSS = 3, +}; + +const char * +db_scan_kind_label(enum scan_kind scan_kind); + +enum scan_kind +db_scan_kind_enum(const char *label); /* Indicates user marked status on a track - values can be bitwise enumerated */ enum usermark { @@ -234,6 +246,8 @@ struct media_file_info { char *album_sort; char *album_artist_sort; char *composer_sort; + + uint32_t scan_kind; /* Identifies the library_source that created/updates this item */ }; #define mfi_offsetof(field) offsetof(struct media_file_info, field) @@ -269,6 +283,7 @@ struct playlist_info { uint32_t query_limit; /* limit, used by e.g. smart playlists */ uint32_t media_kind; char *artwork_url; /* optional artwork */ + uint32_t scan_kind; /* Identifies the library_source that created/updates this item */ uint32_t items; /* number of items (mimc) */ uint32_t streams; /* number of internet streams */ }; @@ -292,6 +307,7 @@ struct db_playlist_info { char *query_limit; char *media_kind; char *artwork_url; + char *scan_kind; char *items; char *streams; }; @@ -405,6 +421,7 @@ struct db_media_file_info { char *composer_sort; char *channels; char *usermark; + char *scan_kind; }; #define dbmfi_offsetof(field) offsetof(struct db_media_file_info, field) @@ -475,6 +492,7 @@ struct directory_info { uint32_t db_timestamp; int64_t disabled; uint32_t parent_id; + uint32_t scan_kind; /* Identifies the library_source that created/updates this item */ }; struct directory_enum { @@ -589,6 +607,9 @@ db_hook_post_scan(void); void db_purge_cruft(time_t ref); +void +db_purge_cruft_bysource(time_t ref, enum scan_kind scan_kind); + void db_purge_all(void); @@ -798,16 +819,19 @@ void db_directory_enum_end(struct directory_enum *de); int -db_directory_addorupdate(char *virtual_path, char *path, int disabled, int parent_id); +db_directory_add(struct directory_info *di, int *id); + +int +db_directory_update(struct directory_info *di); void db_directory_ping_bymatch(char *virtual_path); void -db_directory_disable_bymatch(char *path, enum strip_type strip, uint32_t cookie); +db_directory_disable_bymatch(const char *path, enum strip_type strip, uint32_t cookie); int -db_directory_enable_bycookie(uint32_t cookie, char *path); +db_directory_enable_bycookie(uint32_t cookie, const char *path); int db_directory_enable_bypath(char *path); diff --git a/src/db_init.c b/src/db_init.c index f49d846e..49a07c01 100644 --- a/src/db_init.c +++ b/src/db_init.c @@ -97,7 +97,8 @@ " album_artist_sort VARCHAR(1024) DEFAULT NULL COLLATE DAAP," \ " composer_sort VARCHAR(1024) DEFAULT NULL COLLATE DAAP," \ " channels INTEGER DEFAULT 0," \ - " usermark INTEGER DEFAULT 0" \ + " usermark INTEGER DEFAULT 0," \ + " scan_kind INTEGER DEFAULT 0" \ ");" #define T_PL \ @@ -117,7 +118,8 @@ " query_order VARCHAR(1024)," \ " query_limit INTEGER DEFAULT 0," \ " media_kind INTEGER DEFAULT 1," \ - " artwork_url VARCHAR(4096) DEFAULT NULL" \ + " artwork_url VARCHAR(4096) DEFAULT NULL," \ + " scan_kind INTEGER DEFAULT 0" \ ");" #define T_PLITEMS \ @@ -166,7 +168,8 @@ " db_timestamp INTEGER DEFAULT 0," \ " disabled INTEGER DEFAULT 0," \ " parent_id INTEGER DEFAULT 0," \ - " path VARCHAR(4096) DEFAULT NULL" \ + " path VARCHAR(4096) DEFAULT NULL," \ + " scan_kind INTEGER DEFAULT 0" \ ");" #define T_QUEUE \ diff --git a/src/db_init.h b/src/db_init.h index f196770b..10eac42b 100644 --- a/src/db_init.h +++ b/src/db_init.h @@ -25,8 +25,8 @@ * version of the database? If yes, then it is a minor upgrade, if no, then it * is a major upgrade. In other words minor version upgrades permit downgrading * the server after the database was upgraded. */ -#define SCHEMA_VERSION_MAJOR 21 -#define SCHEMA_VERSION_MINOR 07 +#define SCHEMA_VERSION_MAJOR 22 +#define SCHEMA_VERSION_MINOR 0 int db_init_indices(sqlite3 *hdl); diff --git a/src/db_upgrade.c b/src/db_upgrade.c index f48c99b7..b53abf23 100644 --- a/src/db_upgrade.c +++ b/src/db_upgrade.c @@ -1174,6 +1174,59 @@ static const struct db_upgrade_query db_upgrade_v2107_queries[] = }; +/* ---------------------------- 21.07 -> 22.00 ------------------------------ */ + +#define U_v2200_ALTER_FILES_ADD_SCAN_KIND \ + "ALTER TABLE files ADD COLUMN scan_kind INTEGER DEFAULT 0;" +#define U_v2200_ALTER_PLAYLISTS_ADD_SCAN_KIND \ + "ALTER TABLE playlists ADD COLUMN scan_kind INTEGER DEFAULT 0;" +#define U_v2200_ALTER_DIR_ADD_SCAN_KIND \ + "ALTER TABLE directories ADD COLUMN scan_kind INTEGER DEFAULT 0;" + +#define U_v2200_FILES_SET_SCAN_KIND_RSS \ + "UPDATE files SET scan_kind = 3 WHERE path in (" \ + " SELECT i.filepath from playlists p, playlistitems i WHERE p.id = i.playlistid AND p.type = 4);" +#define U_v2200_FILES_SET_SCAN_KIND_SPOTIFY \ + "UPDATE files SET scan_kind = 2 WHERE virtual_path like '/spotify:/%';" +#define U_v2200_FILES_SET_SOURCE_FILE_SCANNER \ + "UPDATE files SET scan_kind = 1 WHERE scan_kind = 0;" + +#define U_v2200_PL_SET_SCAN_KIND_RSS \ + "UPDATE playlists SET scan_kind = 3 WHERE type = 4;" // PL_RSS = 4 +#define U_v2200_PL_SET_SCAN_KIND_SPOTIFY \ + "UPDATE playlists SET scan_kind = 2 WHERE virtual_path like '/spotify:/%';" +#define U_v2200_PL_SET_SCAN_KIND_FILES \ + "UPDATE playlists SET scan_kind = 1 WHERE scan_kind = 0;" + +// Note: RSS feed items do not have their own directory structure (they use "http:/") +#define U_v2200_DIR_SET_SCAN_KIND_SPOTIFY \ + "UPDATE directories SET scan_kind = 2 WHERE virtual_path like '/spotify:/%';" +#define U_v2200_DIR_SET_SCAN_KIND_FILES \ + "UPDATE directories SET scan_kind = 1 WHERE virtual_path like '/file:/%';" + +#define U_v2200_SCVER_MAJOR \ + "UPDATE admin SET value = '22' WHERE key = 'schema_version_major';" +#define U_v2200_SCVER_MINOR \ + "UPDATE admin SET value = '00' WHERE key = 'schema_version_minor';" + +static const struct db_upgrade_query db_upgrade_v2200_queries[] = + { + { U_v2200_ALTER_FILES_ADD_SCAN_KIND, "alter table files add column scan_kind" }, + { U_v2200_ALTER_PLAYLISTS_ADD_SCAN_KIND, "alter table playlists add column scan_kind" }, + { U_v2200_ALTER_DIR_ADD_SCAN_KIND, "alter table directories add column scan_kind" }, + { U_v2200_FILES_SET_SCAN_KIND_RSS, "update table files set scan_kind rss" }, + { U_v2200_FILES_SET_SCAN_KIND_SPOTIFY, "update table files set scan_kind spotify" }, + { U_v2200_FILES_SET_SOURCE_FILE_SCANNER, "update table files set scan_kind files" }, + { U_v2200_PL_SET_SCAN_KIND_RSS, "update table playlists set scan_kind rss" }, + { U_v2200_PL_SET_SCAN_KIND_SPOTIFY, "update table playlists set scan_kind spotify" }, + { U_v2200_PL_SET_SCAN_KIND_FILES, "update table playlists set scan_kind files" }, + { U_v2200_DIR_SET_SCAN_KIND_SPOTIFY, "update table directories set scan_kind spotify" }, + { U_v2200_DIR_SET_SCAN_KIND_FILES , "update table directories set scan_kind files" }, + + { U_v2200_SCVER_MAJOR, "set schema_version_major to 22" }, + { U_v2200_SCVER_MINOR, "set schema_version_minor to 00" }, + }; + /* -------------------------- Main upgrade handler -------------------------- */ int @@ -1377,6 +1430,13 @@ db_upgrade(sqlite3 *hdl, int db_ver) if (ret < 0) return -1; + /* FALLTHROUGH */ + + case 2107: + ret = db_generic_upgrade(hdl, db_upgrade_v2200_queries, ARRAY_SIZE(db_upgrade_v2200_queries)); + if (ret < 0) + return -1; + /* Last case statement is the only one that ends with a break statement! */ break; diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index 7315e570..5989c440 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -1181,6 +1181,10 @@ jsonapi_reply_library(struct httpd_request *hreq) json_object *jreply; int ret; char *s; + int i; + struct library_source **sources; + json_object *jscanners; + json_object *jsource; CHECK_NULL(L_WEB, jreply = json_object_new_object()); @@ -1216,6 +1220,19 @@ jsonapi_reply_library(struct httpd_request *hreq) json_object_object_add(jreply, "updating", json_object_new_boolean(library_is_scanning())); + jscanners = json_object_new_array(); + json_object_object_add(jreply, "scanners", jscanners); + sources = library_sources(); + for (i = 0; sources[i]; i++) + { + if (!sources[i]->disabled) + { + jsource = json_object_new_object(); + safe_json_add_string(jsource, "name", db_scan_kind_label(sources[i]->scan_kind)); + json_object_array_add(jscanners, jsource); + } + } + CHECK_ERRNO(L_WEB, evbuffer_add_printf(hreq->reply, "%s", json_object_to_json_string(jreply))); jparse_free(jreply); @@ -1228,14 +1245,22 @@ jsonapi_reply_library(struct httpd_request *hreq) static int jsonapi_reply_update(struct httpd_request *hreq) { - library_rescan(); + const char *param; + + param = evhttp_find_header(hreq->query, "scan_kind"); + + library_rescan(db_scan_kind_enum(param)); return HTTP_NOCONTENT; } static int jsonapi_reply_meta_rescan(struct httpd_request *hreq) { - library_metarescan(); + const char *param; + + param = evhttp_find_header(hreq->query, "scan_kind"); + + library_metarescan(db_scan_kind_enum(param)); return HTTP_NOCONTENT; } diff --git a/src/library.c b/src/library.c index 57f31b21..7361c7a9 100644 --- a/src/library.c +++ b/src/library.c @@ -122,10 +122,10 @@ static struct library_callback_register library_cb_register[LIBRARY_MAX_CALLBACK int library_media_save(struct media_file_info *mfi) { - if (!mfi->path || !mfi->fname) + if (!mfi->path || !mfi->fname || !mfi->scan_kind) { - DPRINTF(E_LOG, L_LIB, "Ignoring media file with missing values (path='%s', fname='%s', data_kind='%d')\n", - mfi->path, mfi->fname, mfi->data_kind); + DPRINTF(E_LOG, L_LIB, "Ignoring media file with missing values (path='%s', fname='%s', scan_kind='%d', data_kind='%d')\n", + mfi->path, mfi->fname, mfi->scan_kind, mfi->data_kind); return -1; } @@ -145,9 +145,10 @@ library_media_save(struct media_file_info *mfi) int library_playlist_save(struct playlist_info *pli) { - if (!pli->path) + if (!pli->path || !pli->scan_kind) { - DPRINTF(E_LOG, L_LIB, "Ignoring playlist file with missing path\n"); + DPRINTF(E_LOG, L_LIB, "Ignoring playlist with missing values (path='%s', scan_kind='%d')\n", + pli->path, pli->scan_kind); return -1; } @@ -164,6 +165,39 @@ library_playlist_save(struct playlist_info *pli) return db_pl_update(pli); } +int +library_directory_save(char *virtual_path, char *path, int disabled, int parent_id, enum scan_kind scan_kind) +{ + struct directory_info di = { 0 }; + int id; + int ret; + + id = db_directory_id_byvirtualpath(virtual_path); + + di.id = id; + di.parent_id = parent_id; + di.virtual_path = safe_strdup(virtual_path); + di.path = safe_strdup(path); + di.disabled = disabled; + di.db_timestamp = (uint64_t)time(NULL); + di.scan_kind = scan_kind; + + if (di.id == 0) + ret = db_directory_add(&di, &id); + else + ret = db_directory_update(&di); + + free_di(&di, 1); + + if (ret < 0 || id <= 0) + { + DPRINTF(E_LOG, L_DB, "Insert or update of directory failed '%s'\n", virtual_path); + return -1; + } + + return id; +} + static void scheduled_cb(int fd, short what, void *arg) { @@ -259,20 +293,27 @@ handle_deferred_update_notifications(void) } static void -purge_cruft(time_t start) +purge_cruft(time_t start, enum scan_kind scan_kind) { DPRINTF(E_DBG, L_LIB, "Purging old library content\n"); - db_purge_cruft(start); + if (scan_kind > 0) + db_purge_cruft_bysource(start, scan_kind); + else + db_purge_cruft(start); db_groups_cleanup(); db_queue_cleanup(); - DPRINTF(E_DBG, L_LIB, "Purging old artwork content\n"); - cache_artwork_purge_cruft(start); + if (scan_kind <= 0) + { + DPRINTF(E_DBG, L_LIB, "Purging old artwork content\n"); + cache_artwork_purge_cruft(start); + } } static enum command_state rescan(void *arg, int *ret) { + enum scan_kind *scan_kind; time_t starttime; time_t endtime; int i; @@ -281,20 +322,29 @@ rescan(void *arg, int *ret) listener_notify(LISTENER_UPDATE); starttime = time(NULL); + scan_kind = arg; + for (i = 0; sources[i]; i++) { if (!sources[i]->disabled && sources[i]->rescan) { - DPRINTF(E_INFO, L_LIB, "Rescan library source '%s'\n", sources[i]->name); - sources[i]->rescan(); + if (*scan_kind > 0 && *scan_kind != sources[i]->scan_kind) + { + DPRINTF(E_DBG, L_LIB, "Skipping library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); + } + else + { + DPRINTF(E_INFO, L_LIB, "Rescan library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); + sources[i]->rescan(); + } } else { - DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", sources[i]->name); + DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", db_scan_kind_label(sources[i]->scan_kind)); } } - purge_cruft(starttime); + purge_cruft(starttime, *scan_kind); DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); db_hook_post_scan(); @@ -315,6 +365,7 @@ rescan(void *arg, int *ret) static enum command_state metarescan(void *arg, int *ret) { + enum scan_kind *scan_kind; time_t starttime; time_t endtime; int i; @@ -323,20 +374,29 @@ metarescan(void *arg, int *ret) listener_notify(LISTENER_UPDATE); starttime = time(NULL); + scan_kind = arg; + for (i = 0; sources[i]; i++) { if (!sources[i]->disabled && sources[i]->metarescan) { - DPRINTF(E_INFO, L_LIB, "Meta rescan library source '%s'\n", sources[i]->name); - sources[i]->metarescan(); + if (*scan_kind > 0 && *scan_kind != sources[i]->scan_kind) + { + DPRINTF(E_DBG, L_LIB, "Skipping library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); + } + else + { + DPRINTF(E_INFO, L_LIB, "Meta rescan library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); + sources[i]->metarescan(); + } } else { - DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", sources[i]->name); + DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", db_scan_kind_label(sources[i]->scan_kind)); } } - purge_cruft(starttime); + purge_cruft(starttime, *scan_kind); DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); db_hook_post_scan(); @@ -374,12 +434,12 @@ fullrescan(void *arg, int *ret) { if (!sources[i]->disabled && sources[i]->fullrescan) { - DPRINTF(E_INFO, L_LIB, "Full-rescan library source '%s'\n", sources[i]->name); + DPRINTF(E_INFO, L_LIB, "Full-rescan library source '%s'\n", db_scan_kind_label(sources[i]->scan_kind)); sources[i]->fullrescan(); } else { - DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", sources[i]->name); + DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", db_scan_kind_label(sources[i]->scan_kind)); } } @@ -409,7 +469,7 @@ playlist_item_add(void *arg, int *retval) { if (sources[i]->disabled || !sources[i]->playlist_item_add) { - DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support playlist_item_add\n", sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support playlist_item_add\n", db_scan_kind_label(sources[i]->scan_kind)); continue; } @@ -417,7 +477,7 @@ playlist_item_add(void *arg, int *retval) if (ret == LIBRARY_OK) { - DPRINTF(E_DBG, L_LIB, "Adding item '%s' to playlist '%s' with library source '%s'\n", param->vp_item, param->vp_playlist, sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Adding item '%s' to playlist '%s' with library source '%s'\n", param->vp_item, param->vp_playlist, db_scan_kind_label(sources[i]->scan_kind)); listener_notify(LISTENER_STORED_PLAYLIST); break; } @@ -440,7 +500,7 @@ playlist_remove(void *arg, int *retval) { if (sources[i]->disabled || !sources[i]->playlist_remove) { - DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support playlist_remove\n", sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support playlist_remove\n", db_scan_kind_label(sources[i]->scan_kind)); continue; } @@ -448,7 +508,7 @@ playlist_remove(void *arg, int *retval) if (ret == LIBRARY_OK) { - DPRINTF(E_DBG, L_LIB, "Removing playlist '%s' with library source '%s'\n", virtual_path, sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Removing playlist '%s' with library source '%s'\n", virtual_path, db_scan_kind_label(sources[i]->scan_kind)); listener_notify(LISTENER_STORED_PLAYLIST); break; } @@ -472,7 +532,7 @@ queue_item_add(void *arg, int *retval) { if (sources[i]->disabled || !sources[i]->queue_item_add) { - DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support queue_add\n", sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support queue_add\n", db_scan_kind_label(sources[i]->scan_kind)); continue; } @@ -480,7 +540,7 @@ queue_item_add(void *arg, int *retval) if (ret == LIBRARY_OK) { - DPRINTF(E_DBG, L_LIB, "Items for path '%s' from library source '%s' added to the queue\n", param->path, sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Items for path '%s' from library source '%s' added to the queue\n", param->path, db_scan_kind_label(sources[i]->scan_kind)); break; } } @@ -505,7 +565,7 @@ queue_save(void *arg, int *retval) { if (sources[i]->disabled || !sources[i]->queue_save) { - DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support queue_save\n", sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support queue_save\n", db_scan_kind_label(sources[i]->scan_kind)); continue; } @@ -513,7 +573,7 @@ queue_save(void *arg, int *retval) if (ret == LIBRARY_OK) { - DPRINTF(E_DBG, L_LIB, "Saving queue to path '%s' with library source '%s'\n", virtual_path, sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Saving queue to path '%s' with library source '%s'\n", virtual_path, db_scan_kind_label(sources[i]->scan_kind)); listener_notify(LISTENER_STORED_PLAYLIST); break; } @@ -536,7 +596,7 @@ item_add(void *arg, int *retval) { if (sources[i]->disabled || !sources[i]->item_add) { - DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support add_item\n", sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support add_item\n", db_scan_kind_label(sources[i]->scan_kind)); continue; } @@ -544,7 +604,7 @@ item_add(void *arg, int *retval) if (ret == LIBRARY_OK) { - DPRINTF(E_DBG, L_LIB, "Add item to path '%s' with library source '%s'\n", path, sources[i]->name); + DPRINTF(E_DBG, L_LIB, "Add item to path '%s' with library source '%s'\n", path, db_scan_kind_label(sources[i]->scan_kind)); listener_notify(LISTENER_DATABASE); break; } @@ -597,29 +657,39 @@ update_trigger(void *arg, int *retval) /* ----------------------- LIBRARY EXTERNAL INTERFACE ---------------------- */ void -library_rescan() +library_rescan(enum scan_kind scan_kind) { + int *param; + if (scanning) { DPRINTF(E_INFO, L_LIB, "Scan already running, ignoring request to trigger a new init scan\n"); return; } - scanning = true; // TODO Guard "scanning" with a mutex - commands_exec_async(cmdbase, rescan, NULL); + scanning = true; + param = malloc(sizeof(int)); + *param = scan_kind; + + commands_exec_async(cmdbase, rescan, param); } void -library_metarescan() +library_metarescan(enum scan_kind scan_kind) { + int *param; + if (scanning) { DPRINTF(E_INFO, L_LIB, "Scan already running, ignoring request to trigger metadata scan\n"); return; } - scanning = true; // TODO Guard "scanning" with a mutex - commands_exec_async(cmdbase, metarescan, NULL); + scanning = true; + param = malloc(sizeof(int)); + *param = scan_kind; + + commands_exec_async(cmdbase, metarescan, param); } void @@ -631,7 +701,7 @@ library_fullrescan() return; } - scanning = true; // TODO Guard "scanning" with a mutex + scanning = true; commands_exec_async(cmdbase, fullrescan, NULL); } @@ -662,7 +732,7 @@ initscan() if (! (cfg_getbool(cfg_getsec(cfg, "library"), "filescan_disable"))) { - purge_cruft(starttime); + purge_cruft(starttime, 0); DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); db_hook_post_scan(); @@ -794,11 +864,17 @@ library_item_add(const char *path) return -1; } - scanning = true; // TODO Guard "scanning" with a mutex + scanning = true; return commands_exec_sync(cmdbase, item_add, NULL, (char *)path); } +struct library_source ** +library_sources(void) +{ + return sources; +} + int library_exec_async(command_function func, void *arg) { @@ -863,7 +939,7 @@ library_init(void) { if (!sources[i]->initscan || !sources[i]->rescan || !sources[i]->metarescan || !sources[i]->fullrescan) { - DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' is missing a scanning method\n", sources[i]->name); + DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' is missing a scanning method\n", db_scan_kind_label(sources[i]->scan_kind)); return -1; } diff --git a/src/library.h b/src/library.h index 90b79087..1019a2fc 100644 --- a/src/library.h +++ b/src/library.h @@ -54,7 +54,7 @@ enum library_cb_action */ struct library_source { - char *name; + enum scan_kind scan_kind; int disabled; /* @@ -134,6 +134,9 @@ library_media_save(struct media_file_info *mfi); int library_playlist_save(struct playlist_info *pli); +int +library_directory_save(char *virtual_path, char *path, int disabled, int parent_id, enum scan_kind library_source); + /* * @param cb Callback to call * @param arg Argument to call back with @@ -153,12 +156,27 @@ library_is_exiting(); /* ------------------------ Library external interface --------------------- */ +/* + * Rescan library: find new, remove deleted and update modified tracks and playlists + * If a "source_name" is given, only tracks / playlists belonging to that source are + * updated. + * + * Update is done asynchronously in the library thread. + * + * @param library_source 0 to update everything, one of LIBRARY_SOURCE_xxx to only update specific source + */ void -library_rescan(); +library_rescan(enum scan_kind library_source); +/* + * Same as library_rescan but also updates unmodified tracks and playlists + */ void -library_metarescan(); +library_metarescan(enum scan_kind library_source); +/* + * Wipe library and do a full rescan of all library sources + */ void library_fullrescan(); @@ -201,6 +219,8 @@ library_queue_item_add(const char *path, int position, char reshuffle, uint32_t int library_item_add(const char *path); +struct library_source ** +library_sources(void); /* * Execute the function 'func' with the given argument 'arg' in the library thread. diff --git a/src/library/filescanner.c b/src/library/filescanner.c index 5dc43786..af3598ef 100644 --- a/src/library/filescanner.c +++ b/src/library/filescanner.c @@ -438,6 +438,7 @@ playlist_fill(struct playlist_info *pli, const char *path) pli->path = strdup(path); pli->title = strip_extension(filename); // Will alloc pli->virtual_path = strip_extension(virtual_path); // Will alloc + pli->scan_kind = SCAN_KIND_FILES; pli->directory_id = get_parent_dir_id(path); @@ -586,6 +587,7 @@ process_regular_file(const char *file, struct stat *sb, int type, int flags, int mfi.virtual_path = strdup(virtual_path); mfi.directory_id = dir_id; + mfi.scan_kind = SCAN_KIND_FILES; if (S_ISFIFO(sb->st_mode)) { @@ -699,7 +701,7 @@ process_file(char *file, struct stat *sb, enum file_type file_type, int scan_typ DPRINTF(E_LOG, L_SCAN, "Startup rescan triggered, found init-rescan file: %s\n", file); - library_rescan(); + library_rescan(0); break; case FILE_CTRL_METASCAN: @@ -708,7 +710,7 @@ process_file(char *file, struct stat *sb, enum file_type file_type, int scan_typ DPRINTF(E_LOG, L_SCAN, "Meta rescan triggered, found meta-rescan file: %s\n", file); - library_metarescan(); + library_metarescan(0); break; case FILE_CTRL_FULLSCAN: @@ -828,7 +830,7 @@ process_directory(char *path, int parent_id, int flags) return; } - dir_id = db_directory_addorupdate(virtual_path, path, 0, parent_id); + dir_id = library_directory_save(virtual_path, path, 0, parent_id, SCAN_KIND_FILES); if (dir_id <= 0) { DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '%s'\n", virtual_path); @@ -956,7 +958,7 @@ process_parent_directories(char *path) if (ret < 0) return 0; - dir_id = db_directory_addorupdate(virtual_path, buf, 0, dir_id); + dir_id = library_directory_save(virtual_path, buf, 0, dir_id, SCAN_KIND_FILES); if (dir_id <= 0) { DPRINTF(E_LOG, L_SCAN, "Insert or update of directory failed '%s'\n", virtual_path); @@ -2154,7 +2156,7 @@ filescanner_deinit(void) struct library_source filescanner = { - .name = "filescanner", + .scan_kind = SCAN_KIND_FILES, .disabled = 0, .init = filescanner_init, .deinit = filescanner_deinit, diff --git a/src/library/filescanner_playlist.c b/src/library/filescanner_playlist.c index ca4cd7cb..c7c18471 100644 --- a/src/library/filescanner_playlist.c +++ b/src/library/filescanner_playlist.c @@ -159,6 +159,7 @@ scan_metadata_stream(struct media_file_info *mfi, const char *path) mfi->data_kind = DATA_KIND_HTTP; mfi->time_modified = time(NULL); mfi->directory_id = DIR_HTTP; + mfi->scan_kind = SCAN_KIND_FILES; ret = scan_metadata_ffmpeg(mfi, path); if (ret < 0) @@ -186,6 +187,7 @@ process_nested_playlist(int parent_id, const char *path) goto error; pli->type = PL_FOLDER; + pli->scan_kind = SCAN_KIND_FILES; ret = library_playlist_save(pli); if (ret < 0) goto error; diff --git a/src/library/rssscanner.c b/src/library/rssscanner.c index 0345c7a2..1ed927b9 100644 --- a/src/library/rssscanner.c +++ b/src/library/rssscanner.c @@ -217,6 +217,7 @@ playlist_fetch(bool *is_new, const char *path) pli->directory_id = DIR_HTTP; pli->type = PL_RSS; pli->query_limit = RSS_LIMIT_DEFAULT; + pli->scan_kind = SCAN_KIND_RSS; ret = library_playlist_save(pli); if (ret < 0) @@ -487,6 +488,7 @@ rss_save(struct playlist_info *pli, int *count, enum rss_scan_type scan_type) } scan_metadata_stream(&mfi, ri.url); + mfi.scan_kind = SCAN_KIND_RSS; mfi_metadata_fixup(&mfi, &ri, feed_title, feed_author, time_added); @@ -641,7 +643,7 @@ rss_add(const char *path) struct library_source rssscanner = { - .name = "RSS feeds", + .scan_kind = SCAN_KIND_RSS, .disabled = 0, .initscan = rss_rescan, .rescan = rss_rescan, diff --git a/src/library/spotify_webapi.c b/src/library/spotify_webapi.c index cdeaf3c5..84e80c09 100644 --- a/src/library/spotify_webapi.c +++ b/src/library/spotify_webapi.c @@ -1465,7 +1465,7 @@ prepare_directories(const char *artist, const char *album) DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s)\n", artist); return -1; } - dir_id = db_directory_addorupdate(virtual_path, NULL, 0, DIR_SPOTIFY); + dir_id = library_directory_save(virtual_path, NULL, 0, DIR_SPOTIFY, SCAN_KIND_SPOTIFY); if (dir_id <= 0) { DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); @@ -1477,7 +1477,7 @@ prepare_directories(const char *artist, const char *album) DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s/%s)\n", artist, album); return -1; } - dir_id = db_directory_addorupdate(virtual_path, NULL, 0, dir_id); + dir_id = library_directory_save(virtual_path, NULL, 0, dir_id, SCAN_KIND_SPOTIFY); if (dir_id <= 0) { DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); @@ -1580,6 +1580,7 @@ map_track_to_mfi(struct media_file_info *mfi, const struct spotify_track *track, } snprintf(virtual_path, PATH_MAX, "/spotify:/%s/%s/%s", mfi->album_artist, mfi->album, mfi->title); mfi->virtual_path = strdup(virtual_path); + mfi->scan_kind = SCAN_KIND_SPOTIFY; } static int @@ -1873,8 +1874,9 @@ map_playlist_to_pli(struct playlist_info *pli, struct spotify_playlist *playlist pli->path = strdup(playlist->uri); pli->title = safe_strdup(playlist->name); - pli->parent_id = spotify_base_plid; - pli->directory_id = DIR_SPOTIFY; + pli->parent_id = spotify_base_plid; + pli->directory_id = DIR_SPOTIFY; + pli->scan_kind = SCAN_KIND_SPOTIFY; if (playlist->owner) pli->virtual_path = safe_asprintf("/spotify:/%s (%s)", playlist->name, playlist->owner); @@ -1945,6 +1947,7 @@ create_saved_tracks_playlist(void) .type = PL_PLAIN, .parent_id = spotify_base_plid, .directory_id = DIR_SPOTIFY, + .scan_kind = SCAN_KIND_SPOTIFY, }; spotify_saved_plid = playlist_add_or_update(&pli); @@ -1969,6 +1972,7 @@ create_base_playlist(void) .path = strdup("spotify:playlistfolder"), .title = strdup("Spotify"), .type = PL_FOLDER, + .scan_kind = SCAN_KIND_SPOTIFY, }; spotify_base_plid = 0; @@ -2330,7 +2334,7 @@ spotifywebapi_deinit() struct library_source spotifyscanner = { - .name = "spotifyscanner", + .scan_kind = SCAN_KIND_SPOTIFY, .disabled = 0, .init = spotifywebapi_init, .deinit = spotifywebapi_deinit, diff --git a/src/mpd.c b/src/mpd.c index 58929e8d..e31dc392 100644 --- a/src/mpd.c +++ b/src/mpd.c @@ -3203,7 +3203,7 @@ mpd_command_update(struct evbuffer *evbuf, int argc, char **argv, char **errmsg, return ACK_ERROR_ARG; } - library_rescan(); + library_rescan(0); evbuffer_add(evbuf, "updating_db: 1\n", 15);