From d2d85b29f4203bc3ab3ac52a60bb92e4acd98d5e Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Sun, 7 Sep 2014 13:48:03 +0200 Subject: [PATCH 1/4] Lower the priority of the thread so forked-daapd may still respond during file scan on low power devices --- src/filescanner.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/filescanner.c b/src/filescanner.c index c9a6bfcf..26817663 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -1074,8 +1074,20 @@ bulk_scan(int flags) static void * filescanner(void *arg) { + struct sched_param param; int ret; + /* Lower the priority of the thread so forked-daapd may still respond + * during file scan on low power devices. Param must be 0 for the SCHED_BATCH + * policy. + */ + memset(¶m, 0, sizeof(struct sched_param)); + ret = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (ret != 0) + { + DPRINTF(E_LOG, L_SCAN, "Warning: Could not set thread priority to SCHED_BATCH\n"); + } + ret = db_perthread_init(); if (ret < 0) { From 94c5352db69d2bd0e5f7df6ab4b7379e84da79de Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Mon, 8 Sep 2014 21:57:52 +0200 Subject: [PATCH 2/4] Remove EVLOOP_ONCE calls in filescanner These calls generate error messages when the loop is already running, which will be the case when using init-rescan or full-rescan. The only purpose of these calls seems to be to check for exit signals from the main thread during a startup bulk scan, where the loop is not running yet. However, we can check for an exit signal by just setting/checking scan_exit. This commit also removes the unused filescanner_status(). --- src/filescanner.c | 34 +++++----------------------------- src/filescanner.h | 3 --- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 26817663..8440c86c 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -656,9 +656,6 @@ process_deferred_playlists(void) free(pl->path); free(pl); - /* Run the event loop */ - event_base_loop(evbase_scan, EVLOOP_ONCE | EVLOOP_NONBLOCK); - if (scan_exit) return; } @@ -782,7 +779,6 @@ check_speciallib(char *path, const char *libtype) static void process_directory(char *path, int flags) { - struct stacked_dir *bulkstack; DIR *dirp; struct dirent buf; struct dirent *de; @@ -796,24 +792,6 @@ process_directory(char *path, int flags) int type; int ret; - if (flags & F_SCAN_BULK) - { - /* Save our directory stack so it won't get handled inside - * the event loop - not its business, we're in bulk mode here. - */ - bulkstack = dirstack; - dirstack = NULL; - - /* Run the event loop */ - event_base_loop(evbase_scan, EVLOOP_ONCE | EVLOOP_NONBLOCK); - - /* Restore our directory stack */ - dirstack = bulkstack; - - if (scan_exit) - return; - } - DPRINTF(E_DBG, L_SCAN, "Processing directory %s (flags = 0x%x)\n", path, flags); dirp = opendir(path); @@ -835,6 +813,9 @@ process_directory(char *path, int flags) for (;;) { + if (scan_exit) + break; + ret = readdir_r(dirp, &buf, &de); if (ret != 0) { @@ -1799,13 +1780,6 @@ exit_cb(int fd, short event, void *arg) scan_exit = 1; } - -int -filescanner_status(void) -{ - return scan_exit; -} - /* Thread: main */ int filescanner_init(void) @@ -1909,6 +1883,8 @@ filescanner_deinit(void) } #endif + scan_exit = 1; + ret = pthread_join(tid_scan, NULL); if (ret != 0) { diff --git a/src/filescanner.h b/src/filescanner.h index 07c8ea9b..97c628e8 100644 --- a/src/filescanner.h +++ b/src/filescanner.h @@ -18,9 +18,6 @@ filescanner_init(void); void filescanner_deinit(void); -int -filescanner_status(void); - void filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi); From be7a6c7b1ea59310ea4e38837fa1bbdac2357db8 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Mon, 8 Sep 2014 22:45:05 +0200 Subject: [PATCH 3/4] Don't try to login to Spotify if the filescanner got an exit signal --- src/filescanner.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 8440c86c..4736b5dc 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -1105,12 +1105,12 @@ filescanner(void *arg) else bulk_scan(F_SCAN_BULK); -#ifdef HAVE_SPOTIFY_H - spotify_login(NULL); -#endif - if (!scan_exit) { +#ifdef HAVE_SPOTIFY_H + spotify_login(NULL); +#endif + /* Enable inotify */ event_add(&inoev, NULL); From 5bfe4673f5070b2637241e9cfb6ccb29f32fc4c0 Mon Sep 17 00:00:00 2001 From: ejurgensen Date: Wed, 10 Sep 2014 22:16:52 +0200 Subject: [PATCH 4/4] Add a function in filescanner.c to enumerate certain file types Fixes bug where init-rescan and full-rescan would run twice because two inotify events get triggered by eg 'touch xxx.init-rescan' --- src/filescanner.c | 254 +++++++++++++++++++++++++--------------------- 1 file changed, 139 insertions(+), 115 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 4736b5dc..86f3af29 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -76,6 +76,20 @@ #define F_SCAN_FAST (1 << 2) #define F_SCAN_MOVED (1 << 3) +enum file_type { + FILE_UNKNOWN = 0, + FILE_IGNORE, + FILE_REGULAR, + FILE_PLAYLIST, + FILE_ITUNES, + FILE_ARTWORK, + FILE_CTRL_REMOTE, + FILE_CTRL_LASTFM, + FILE_CTRL_SPOTIFY, + FILE_CTRL_INITSCAN, + FILE_CTRL_FULLSCAN, +}; + struct deferred_pl { char *path; time_t mtime; @@ -156,8 +170,9 @@ pop_dir(struct stacked_dir **s) return ret; } +/* Checks if the file extension is in the ignore list */ static int -ignore_filetype(char *ext) +file_type_ignore(const char *ext) { cfg_t *lib; int n; @@ -175,6 +190,66 @@ ignore_filetype(char *ext) return 0; } +static enum file_type +file_type_get(const char *path) { + const char *filename; + const char *ext; + + filename = strrchr(path, '/'); + if ((!filename) || (strlen(filename) == 1)) + filename = path; + else + filename++; + + ext = strrchr(path, '.'); + if (!ext || (strlen(ext) == 1)) + return FILE_REGULAR; + + if ((strcasecmp(ext, ".m3u") == 0) || (strcasecmp(ext, ".pls") == 0)) + return FILE_PLAYLIST; + + if ((strcasecmp(ext, ".png") == 0) || (strcasecmp(ext, ".jpg") == 0)) + return FILE_ARTWORK; + +#ifdef ITUNES + if (strcasecmp(ext, ".xml") == 0) + return FILE_ITUNES; +#endif + + if (strcasecmp(ext, ".remote") == 0) + return FILE_CTRL_REMOTE; + +#ifdef LASTFM + if (strcasecmp(ext, ".lastfm") == 0) + return FILE_CTRL_LASTFM; +#endif + +#ifdef HAVE_SPOTIFY_H + if (strcasecmp(ext, ".spotify") == 0) + return FILE_CTRL_SPOTIFY; +#endif + + if (strcasecmp(ext, ".init-rescan") == 0) + return FILE_CTRL_INITSCAN; + + if (strcasecmp(ext, ".full-rescan") == 0) + return FILE_CTRL_FULLSCAN; + + if (strcasecmp(ext, ".url") == 0) + { + DPRINTF(E_INFO, L_SCAN, "No support for .url, use .m3u or .pls\n"); + return FILE_IGNORE; + } + + if (file_type_ignore(ext)) + return FILE_IGNORE; + + if ((filename[0] == '_') || (filename[0] == '.')) + return FILE_IGNORE; + + return FILE_REGULAR; +} + static void sort_tag_create(char **sort_tag, char *src_tag) { @@ -445,7 +520,6 @@ filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct { struct media_file_info *mfi; char *filename; - char *ext; time_t stamp; int id; int ret; @@ -456,33 +530,6 @@ filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct else filename++; - /* File types which should never be processed */ - ext = strrchr(path, '.'); - if (ext) - { - if (strcasecmp(ext, ".url") == 0) - { - DPRINTF(E_INFO, L_SCAN, "No support for .url in this version, use .m3u or .pls\n"); - - return; - } - else if ((strcasecmp(ext, ".png") == 0) || (strcasecmp(ext, ".jpg") == 0)) - { - /* Artwork files - don't scan */ - return; - } - else if ((filename[0] == '_') || (filename[0] == '.')) - { - /* Hidden files - don't scan */ - return; - } - else if (ignore_filetype(ext)) - { - /* File extension is in ignore list - don't scan */ - return; - } - } - db_file_stamp_bypath(path, &stamp, &id); if (stamp && (stamp >= mtime)) @@ -593,20 +640,15 @@ filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct static void process_playlist(char *file, time_t mtime) { - char *ext; + enum file_type ft; - ext = strrchr(file, '.'); - if (ext) - { - if (strcasecmp(ext, ".m3u") == 0) - scan_playlist(file, mtime); - else if (strcasecmp(ext, ".pls") == 0) - scan_playlist(file, mtime); + ft = file_type_get(file); + if (ft == FILE_PLAYLIST) + scan_playlist(file, mtime); #ifdef ITUNES - else if (strcasecmp(ext, ".xml") == 0) - scan_itunes_itml(file); + else if (ft == FILE_ITUNES) + scan_itunes_itml(file); #endif - } } /* Thread: scan */ @@ -665,94 +707,76 @@ process_deferred_playlists(void) static void process_file(char *file, time_t mtime, off_t size, int type, int flags) { - char *ext; - - ext = strrchr(file, '.'); - if (ext) + switch (file_type_get(file)) { - if ((strcasecmp(ext, ".m3u") == 0) - || (strcasecmp(ext, ".pls") == 0) -#ifdef ITUNES - || (strcasecmp(ext, ".xml") == 0) -#endif - ) - { - if (flags & F_SCAN_BULK) - defer_playlist(file, mtime); - else - process_playlist(file, mtime); + case FILE_REGULAR: + filescanner_process_media(file, mtime, size, type, NULL); - return; - } - else if (strcmp(ext, ".remote") == 0) - { - remote_pairing_read_pin(file); + counter++; + + /* When in bulk mode, split transaction in pieces of 200 */ + if ((flags & F_SCAN_BULK) && (counter % 200 == 0)) + { + DPRINTF(E_LOG, L_SCAN, "Scanned %d files...\n", counter); + db_transaction_end(); + db_transaction_begin(); + } + break; + + case FILE_PLAYLIST: + case FILE_ITUNES: + if (flags & F_SCAN_BULK) + defer_playlist(file, mtime); + else + process_playlist(file, mtime); + break; + + case FILE_CTRL_REMOTE: + remote_pairing_read_pin(file); + break; - return; - } #ifdef LASTFM - else if (strcmp(ext, ".lastfm") == 0) - { - lastfm_login(file); - - return; - } + case FILE_CTRL_LASTFM: + lastfm_login(file); + break; #endif + #ifdef HAVE_SPOTIFY_H - else if (strcmp(ext, ".spotify") == 0) - { - spotify_login(file); - - return; - } + case FILE_CTRL_SPOTIFY: + spotify_login(file); + break; #endif - else if (strcmp(ext, ".full-rescan") == 0) - { - if (flags & F_SCAN_BULK) - return; - else - { - DPRINTF(E_LOG, L_SCAN, "Forcing full rescan, found full-rescan file: %s\n", file); - player_playback_stop(); - player_queue_clear(); - inofd_event_unset(); // Clears all inotify watches - db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups - inofd_event_set(); - bulk_scan(F_SCAN_BULK); + case FILE_CTRL_INITSCAN: + if (flags & F_SCAN_BULK) + break; - return; - } - } - else if (strcmp(ext, ".init-rescan") == 0) - { - if (flags & F_SCAN_BULK) - return; - else - { - DPRINTF(E_LOG, L_SCAN, "Forcing startup rescan, found init-rescan file: %s\n", file); - inofd_event_unset(); // Clears all inotify watches - db_watch_clear(); + DPRINTF(E_LOG, L_SCAN, "Startup rescan triggered, found init-rescan file: %s\n", file); - inofd_event_set(); - bulk_scan(F_SCAN_BULK | F_SCAN_RESCAN); + inofd_event_unset(); // Clears all inotify watches + db_watch_clear(); - return; - } - } - } + inofd_event_set(); + bulk_scan(F_SCAN_BULK | F_SCAN_RESCAN); + break; - /* Not any kind of special file, so let's see if it's a media file */ - filescanner_process_media(file, mtime, size, type, NULL); + case FILE_CTRL_FULLSCAN: + if (flags & F_SCAN_BULK) + break; - counter++; + DPRINTF(E_LOG, L_SCAN, "Full rescan triggered, found full-rescan file: %s\n", file); - /* When in bulk mode, split transaction in pieces of 200 */ - if ((flags & F_SCAN_BULK) && (counter % 200 == 0)) - { - DPRINTF(E_LOG, L_SCAN, "Scanned %d files...\n", counter); - db_transaction_end(); - db_transaction_begin(); + player_playback_stop(); + player_queue_clear(); + inofd_event_unset(); // Clears all inotify watches + db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups + + inofd_event_set(); + bulk_scan(F_SCAN_BULK); + break; + + default: + DPRINTF(E_WARN, L_SCAN, "Ignoring file: %s\n", file); } } @@ -1333,7 +1357,7 @@ process_inotify_file(struct watch_info *wi, char *path, struct inotify_event *ie db_file_delete_bypath(path);; } - else if (db_file_id_bypath(path) <= 0) + else if ((file_type_get(path) == FILE_REGULAR) && (db_file_id_bypath(path) <= 0)) // TODO Playlists { DPRINTF(E_LOG, L_SCAN, "File access to '%s' achieved\n", path);