From 56ce3f9cbab4ece0f9db43286512af3030e8203a Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 31 Dec 2016 07:28:18 +0100 Subject: [PATCH 01/33] [library/filescanner/spotify] Add library source abstraction and new "library" thread - Implement filescanner as a library_source - Add library source implementation for spotify - Move process_media_info to library with small adjustments: - Remove dependency to F_SCAN_TYPE defines - Pass data_kind as parameter - Pass media_kind and compilation as parameter if a source forces a specific value for these - Move declaration of scan_ffmpeg to library; add library_add_playlist_info to library --- src/Makefile.am | 1 + src/db.c | 6 +- src/db.h | 6 +- src/filescanner.c | 693 ++++---------------------------- src/filescanner.h | 20 - src/filescanner_ffmpeg.c | 2 +- src/filescanner_playlist.c | 3 +- src/library.c | 782 +++++++++++++++++++++++++++++++++++++ src/library.h | 98 +++++ src/logger.c | 2 +- src/logger.h | 3 +- src/main.c | 32 +- src/mpd.c | 6 +- src/spotify.c | 38 +- 14 files changed, 1018 insertions(+), 674 deletions(-) create mode 100644 src/library.c create mode 100644 src/library.h diff --git a/src/Makefile.am b/src/Makefile.am index ca9027f6..71d21290 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -82,6 +82,7 @@ forked_daapd_SOURCES = main.c \ filescanner.c filescanner.h \ filescanner_ffmpeg.c filescanner_playlist.c \ filescanner_smartpl.c $(ITUNES_SRC) \ + library.c library.h \ mdns_avahi.c mdns.h \ remote_pairing.c remote_pairing.h \ avio_evbuffer.c avio_evbuffer.h \ diff --git a/src/db.c b/src/db.c index 126785fe..9bcf0453 100644 --- a/src/db.c +++ b/src/db.c @@ -2084,7 +2084,7 @@ db_file_id_by_virtualpath_match(char *path) } void -db_file_stamp_bypath(char *path, time_t *stamp, int *id) +db_file_stamp_bypath(const char *path, time_t *stamp, int *id) { #define Q_TMPL "SELECT f.id, f.db_timestamp FROM files f WHERE f.path = '%q';" char *query; @@ -2847,7 +2847,7 @@ db_pl_fetch_byquery(char *query) } struct playlist_info * -db_pl_fetch_bypath(char *path) +db_pl_fetch_bypath(const char *path) { #define Q_TMPL "SELECT p.* FROM playlists p WHERE p.path = '%q';" struct playlist_info *pli; @@ -3011,7 +3011,7 @@ db_pl_add(struct playlist_info *pli, int *id) } int -db_pl_add_item_bypath(int plid, char *path) +db_pl_add_item_bypath(int plid, const char *path) { #define Q_TMPL "INSERT INTO playlistitems (playlistid, filepath) VALUES (%d, '%q');" char *query; diff --git a/src/db.h b/src/db.h index 4ecf43c0..6e2c21b3 100644 --- a/src/db.h +++ b/src/db.h @@ -527,7 +527,7 @@ int db_file_id_by_virtualpath_match(char *path); void -db_file_stamp_bypath(char *path, time_t *stamp, int *id); +db_file_stamp_bypath(const char *path, time_t *stamp, int *id); struct media_file_info * db_file_fetch_byid(int id); @@ -570,7 +570,7 @@ void db_pl_ping_bymatch(char *path, int isdir); struct playlist_info * -db_pl_fetch_bypath(char *path); +db_pl_fetch_bypath(const char *path); struct playlist_info * db_pl_fetch_byvirtualpath(char *virtual_path); @@ -582,7 +582,7 @@ int db_pl_add(struct playlist_info *pli, int *id); int -db_pl_add_item_bypath(int plid, char *path); +db_pl_add_item_bypath(int plid, const char *path); int db_pl_add_item_byid(int plid, int fileid); diff --git a/src/filescanner.c b/src/filescanner.c index 5779ffd4..2b526c50 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -62,6 +62,7 @@ #include "cache.h" #include "artwork.h" #include "commands.h" +#include "library.h" #ifdef LASTFM # include "lastfm.h" @@ -104,14 +105,14 @@ struct stacked_dir { struct stacked_dir *next; }; -static int scan_exit; static int inofd; -static struct event_base *evbase_scan; static struct event *inoev; -static pthread_t tid_scan; static struct deferred_pl *playlists; static struct stacked_dir *dirstack; -static struct commands_base *cmdbase; + +/* From library.c */ +extern struct event_base *evbase_lib; + #ifndef __linux__ struct deferred_file @@ -131,9 +132,6 @@ static struct event *deferred_inoev; /* Count of files scanned during a bulk scan */ static int counter; -/* Flag for scan in progress */ -static int scanning; - /* When copying into the lib (eg. if a file is moved to the lib by copying into * a Samba network share) inotify might give us IN_CREATE -> n x IN_ATTRIB -> * IN_CLOSE_WRITE, but we don't want to do any scanning before the @@ -151,10 +149,12 @@ static int inofd_event_set(void); static void inofd_event_unset(void); -static enum command_state -filescanner_initscan(void *arg, int *retval); -static enum command_state -filescanner_fullrescan(void *arg, int *retval); +static int +filescanner_initscan(); +static int +filescanner_rescan(); +static int +filescanner_fullrescan(); static int @@ -323,416 +323,6 @@ file_type_get(const char *path) { return FILE_REGULAR; } -static void -sort_tag_create(char **sort_tag, char *src_tag) -{ - const uint8_t *i_ptr; - const uint8_t *n_ptr; - const uint8_t *number; - uint8_t out[1024]; - uint8_t *o_ptr; - int append_number; - ucs4_t puc; - int numlen; - size_t len; - int charlen; - - /* Note: include terminating NUL in string length for u8_normalize */ - - if (*sort_tag) - { - DPRINTF(E_DBG, L_SCAN, "Existing sort tag will be normalized: %s\n", *sort_tag); - o_ptr = u8_normalize(UNINORM_NFD, (uint8_t *)*sort_tag, strlen(*sort_tag) + 1, NULL, &len); - free(*sort_tag); - *sort_tag = (char *)o_ptr; - return; - } - - if (!src_tag || ((len = strlen(src_tag)) == 0)) - { - *sort_tag = NULL; - return; - } - - // Set input pointer past article if present - if ((strncasecmp(src_tag, "a ", 2) == 0) && (len > 2)) - i_ptr = (uint8_t *)(src_tag + 2); - else if ((strncasecmp(src_tag, "an ", 3) == 0) && (len > 3)) - i_ptr = (uint8_t *)(src_tag + 3); - else if ((strncasecmp(src_tag, "the ", 4) == 0) && (len > 4)) - i_ptr = (uint8_t *)(src_tag + 4); - else - i_ptr = (uint8_t *)src_tag; - - // Poor man's natural sort. Makes sure we sort like this: a1, a2, a10, a11, a21, a111 - // We do this by padding zeroes to (short) numbers. As an alternative we could have - // made a proper natural sort algorithm in sqlext.c, but we don't, since we don't - // want any risk of hurting response times - memset(&out, 0, sizeof(out)); - o_ptr = (uint8_t *)&out; - number = NULL; - append_number = 0; - - do - { - n_ptr = u8_next(&puc, i_ptr); - - if (uc_is_digit(puc)) - { - if (!number) // We have encountered the beginning of a number - number = i_ptr; - append_number = (n_ptr == NULL); // If last char in string append number now - } - else - { - if (number) - append_number = 1; // A number has ended so time to append it - else - { - charlen = u8_strmblen(i_ptr); - if (charlen >= 0) - o_ptr = u8_stpncpy(o_ptr, i_ptr, charlen); // No numbers in sight, just append char - } - } - - // Break if less than 100 bytes remain (prevent buffer overflow) - if (sizeof(out) - u8_strlen(out) < 100) - break; - - // Break if number is very large (prevent buffer overflow) - if (number && (i_ptr - number > 50)) - break; - - if (append_number) - { - numlen = i_ptr - number; - if (numlen < 5) // Max pad width - { - u8_strcpy(o_ptr, (uint8_t *)"00000"); - o_ptr += (5 - numlen); - } - o_ptr = u8_stpncpy(o_ptr, number, numlen + u8_strmblen(i_ptr)); - - number = NULL; - append_number = 0; - } - - i_ptr = n_ptr; - } - while (n_ptr); - - *sort_tag = (char *)u8_normalize(UNINORM_NFD, (uint8_t *)&out, u8_strlen(out) + 1, NULL, &len); -} - -static void -fixup_tags(struct media_file_info *mfi) -{ - cfg_t *lib; - size_t len; - char *tag; - char *sep = " - "; - char *ca; - - if (mfi->genre && (strlen(mfi->genre) == 0)) - { - free(mfi->genre); - mfi->genre = NULL; - } - - if (mfi->artist && (strlen(mfi->artist) == 0)) - { - free(mfi->artist); - mfi->artist = NULL; - } - - if (mfi->title && (strlen(mfi->title) == 0)) - { - free(mfi->title); - mfi->title = NULL; - } - - /* - * Default to mpeg4 video/audio for unknown file types - * in an attempt to allow streaming of DRM-afflicted files - */ - if (mfi->codectype && strcmp(mfi->codectype, "unkn") == 0) - { - if (mfi->has_video) - { - strcpy(mfi->codectype, "mp4v"); - strcpy(mfi->type, "m4v"); - } - else - { - strcpy(mfi->codectype, "mp4a"); - strcpy(mfi->type, "m4a"); - } - } - - if (!mfi->artist) - { - if (mfi->orchestra && mfi->conductor) - { - len = strlen(mfi->orchestra) + strlen(sep) + strlen(mfi->conductor); - tag = (char *)malloc(len + 1); - if (tag) - { - sprintf(tag,"%s%s%s", mfi->orchestra, sep, mfi->conductor); - mfi->artist = tag; - } - } - else if (mfi->orchestra) - { - mfi->artist = strdup(mfi->orchestra); - } - else if (mfi->conductor) - { - mfi->artist = strdup(mfi->conductor); - } - } - - /* Handle TV shows, try to present prettier metadata */ - if (mfi->tv_series_name && strlen(mfi->tv_series_name) != 0) - { - mfi->media_kind = MEDIA_KIND_TVSHOW; /* tv show */ - - /* Default to artist = series_name */ - if (mfi->artist && strlen(mfi->artist) == 0) - { - free(mfi->artist); - mfi->artist = NULL; - } - - if (!mfi->artist) - mfi->artist = strdup(mfi->tv_series_name); - - /* Default to album = ", Season " */ - if (mfi->album && strlen(mfi->album) == 0) - { - free(mfi->album); - mfi->album = NULL; - } - - if (!mfi->album) - { - len = snprintf(NULL, 0, "%s, Season %d", mfi->tv_series_name, mfi->tv_season_num); - - mfi->album = (char *)malloc(len + 1); - if (mfi->album) - sprintf(mfi->album, "%s, Season %d", mfi->tv_series_name, mfi->tv_season_num); - } - } - - /* Check the 4 top-tags are filled */ - if (!mfi->artist) - mfi->artist = strdup("Unknown artist"); - if (!mfi->album) - mfi->album = strdup("Unknown album"); - if (!mfi->genre) - mfi->genre = strdup("Unknown genre"); - if (!mfi->title) - { - /* fname is left untouched by unicode_fixup_mfi() for - * obvious reasons, so ensure it is proper UTF-8 - */ - mfi->title = unicode_fixup_string(mfi->fname, "ascii"); - if (mfi->title == mfi->fname) - mfi->title = strdup(mfi->fname); - } - - /* Ensure sort tags are filled, manipulated and normalized */ - sort_tag_create(&mfi->artist_sort, mfi->artist); - sort_tag_create(&mfi->album_sort, mfi->album); - sort_tag_create(&mfi->title_sort, mfi->title); - - /* We need to set album_artist according to media type and config */ - if (mfi->compilation) /* Compilation */ - { - lib = cfg_getsec(cfg, "library"); - ca = cfg_getstr(lib, "compilation_artist"); - if (ca && mfi->album_artist) - { - free(mfi->album_artist); - mfi->album_artist = strdup(ca); - } - else if (ca && !mfi->album_artist) - { - mfi->album_artist = strdup(ca); - } - else if (!ca && !mfi->album_artist) - { - mfi->album_artist = strdup(""); - mfi->album_artist_sort = strdup(""); - } - } - else if (mfi->media_kind == MEDIA_KIND_PODCAST) /* Podcast */ - { - if (mfi->album_artist) - free(mfi->album_artist); - mfi->album_artist = strdup(""); - mfi->album_artist_sort = strdup(""); - } - else if (!mfi->album_artist) /* Regular media without album_artist */ - { - mfi->album_artist = strdup(mfi->artist); - } - - if (!mfi->album_artist_sort && (strcmp(mfi->album_artist, mfi->artist) == 0)) - mfi->album_artist_sort = strdup(mfi->artist_sort); - else - sort_tag_create(&mfi->album_artist_sort, mfi->album_artist); - - /* Composer is not one of our mandatory tags, so take extra care */ - if (mfi->composer_sort || mfi->composer) - sort_tag_create(&mfi->composer_sort, mfi->composer); -} - - -void -filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi, int dir_id) -{ - struct media_file_info *mfi; - char *filename; - time_t stamp; - int id; - char virtual_path[PATH_MAX]; - int ret; - - filename = strrchr(path, '/'); - if ((!filename) || (strlen(filename) == 1)) - filename = path; - else - filename++; - - db_file_stamp_bypath(path, &stamp, &id); - - if (stamp && (stamp >= mtime)) - { - db_file_ping(id); - return; - } - - if (!external_mfi) - { - mfi = (struct media_file_info*)malloc(sizeof(struct media_file_info)); - if (!mfi) - { - DPRINTF(E_LOG, L_SCAN, "Out of memory for mfi\n"); - return; - } - - memset(mfi, 0, sizeof(struct media_file_info)); - } - else - mfi = external_mfi; - - if (stamp) - mfi->id = id; - - mfi->fname = strdup(filename); - if (!mfi->fname) - { - DPRINTF(E_LOG, L_SCAN, "Out of memory for fname\n"); - goto out; - } - - mfi->path = strdup(path); - if (!mfi->path) - { - DPRINTF(E_LOG, L_SCAN, "Out of memory for path\n"); - goto out; - } - - mfi->time_modified = mtime; - mfi->file_size = size; - - if (type & F_SCAN_TYPE_COMPILATION) - mfi->compilation = 1; - if (type & F_SCAN_TYPE_PODCAST) - mfi->media_kind = MEDIA_KIND_PODCAST; /* podcast */ - if (type & F_SCAN_TYPE_AUDIOBOOK) - mfi->media_kind = MEDIA_KIND_AUDIOBOOK; /* audiobook */ - - if (type & F_SCAN_TYPE_FILE) - { - mfi->data_kind = DATA_KIND_FILE; - ret = scan_metadata_ffmpeg(path, mfi); - } - else if (type & F_SCAN_TYPE_URL) - { - mfi->data_kind = DATA_KIND_HTTP; - ret = scan_metadata_ffmpeg(path, mfi); - if (ret < 0) - { - DPRINTF(E_LOG, L_SCAN, "Playlist URL '%s' is unavailable for probe/metadata, assuming MP3 encoding\n", path); - mfi->type = strdup("mp3"); - mfi->codectype = strdup("mpeg"); - mfi->description = strdup("MPEG audio file"); - ret = 1; - } - } - else if (type & F_SCAN_TYPE_SPOTIFY) - { - mfi->data_kind = DATA_KIND_SPOTIFY; - ret = mfi->artist && mfi->album && mfi->title; - } - else if (type & F_SCAN_TYPE_PIPE) - { - mfi->data_kind = DATA_KIND_PIPE; - mfi->type = strdup("wav"); - mfi->codectype = strdup("wav"); - mfi->description = strdup("PCM16 pipe"); - ret = 1; - } - else - { - DPRINTF(E_LOG, L_SCAN, "Unknown scan type for '%s', this error should not occur\n", path); - ret = -1; - } - - if (ret < 0) - { - DPRINTF(E_INFO, L_SCAN, "Could not extract metadata for '%s'\n", path); - goto out; - } - - if (!mfi->item_kind) - mfi->item_kind = 2; /* music */ - if (!mfi->media_kind) - mfi->media_kind = MEDIA_KIND_MUSIC; /* music */ - - unicode_fixup_mfi(mfi); - - fixup_tags(mfi); - - if (type & F_SCAN_TYPE_URL) - { - snprintf(virtual_path, PATH_MAX, "/http:/%s", mfi->title); - mfi->virtual_path = strdup(virtual_path); - } - else if (type & F_SCAN_TYPE_SPOTIFY) - { - snprintf(virtual_path, PATH_MAX, "/spotify:/%s/%s/%s", mfi->album_artist, mfi->album, mfi->title); - mfi->virtual_path = strdup(virtual_path); - } - else - { - snprintf(virtual_path, PATH_MAX, "/file:%s", mfi->path); - mfi->virtual_path = strdup(virtual_path); - } - - mfi->directory_id = dir_id; - - if (mfi->id == 0) - db_file_add(mfi); - else - db_file_update(mfi); - - out: - if (!external_mfi) - free_mfi(mfi, 0); -} - static void process_playlist(char *file, time_t mtime, int dir_id) { @@ -795,7 +385,7 @@ process_deferred_playlists(void) free(pl->path); free(pl); - if (scan_exit) + if (library_is_exiting()) return; } } @@ -804,15 +394,24 @@ process_deferred_playlists(void) static void process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_id) { - int is_bulkscan; - int ret; + bool is_bulkscan; + bool is_type_compilation; + enum media_kind media_kind; is_bulkscan = (flags & F_SCAN_BULK); switch (file_type_get(file)) { case FILE_REGULAR: - filescanner_process_media(file, mtime, size, type, NULL, dir_id); + is_type_compilation = type & F_SCAN_TYPE_COMPILATION; + if (type & F_SCAN_TYPE_AUDIOBOOK) + media_kind = MEDIA_KIND_AUDIOBOOK; + else if (type & F_SCAN_TYPE_PODCAST) + media_kind = MEDIA_KIND_PODCAST; + else + media_kind = 0; + + library_process_media(file, mtime, size, DATA_KIND_FILE, media_kind, is_type_compilation, NULL, dir_id); cache_artwork_ping(file, mtime, !is_bulkscan); // TODO [artworkcache] If entry in artwork cache exists for no artwork available, delete the entry if media file has embedded artwork @@ -884,7 +483,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ DPRINTF(E_LOG, L_SCAN, "Startup rescan triggered, found init-rescan file: %s\n", file); - filescanner_initscan(NULL, &ret); + filescanner_rescan(); break; case FILE_CTRL_FULLSCAN: @@ -893,7 +492,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ DPRINTF(E_LOG, L_SCAN, "Full rescan triggered, found full-rescan file: %s\n", file); - filescanner_fullrescan(NULL, &ret); + filescanner_fullrescan(); break; default: @@ -982,7 +581,7 @@ process_directory(char *path, int parent_id, int flags) for (;;) { - if (scan_exit) + if (library_is_exiting()) break; errno = 0; @@ -1139,7 +738,7 @@ process_directories(char *root, int parent_id, int flags) process_directory(root, parent_id, flags); - if (scan_exit) + if (library_is_exiting()) return; while ((dir = pop_dir(&dirstack))) @@ -1149,7 +748,7 @@ process_directories(char *root, int parent_id, int flags) free(dir->path); free(dir); - if (scan_exit) + if (library_is_exiting()) return; } } @@ -1168,9 +767,6 @@ bulk_scan(int flags) int parent_id; int i; - // Set global flag to avoid queued scan requests - scanning = 1; - start = time(NULL); playlists = NULL; @@ -1210,14 +806,14 @@ bulk_scan(int flags) free(deref); - if (scan_exit) + if (library_is_exiting()) return; } if (!(flags & F_SCAN_FAST) && playlists) process_deferred_playlists(); - if (scan_exit) + if (library_is_exiting()) return; if (dirstack) @@ -1249,85 +845,6 @@ bulk_scan(int flags) DPRINTF(E_DBG, L_SCAN, "Running post library scan jobs\n"); db_hook_post_scan(); } - - // Set scan in progress flag to FALSE - scanning = 0; -} - - -/* Thread: scan */ -static void * -filescanner(void *arg) -{ - int clear_queue_on_stop_disabled; - int ret; -#ifdef __linux__ - struct sched_param param; - - /* Lower the priority of the thread so forked-daapd may still respond - * during file scan on low power devices. Param must be 0 for the SCHED_BATCH - * policy. - */ - memset(¶m, 0, sizeof(struct sched_param)); - ret = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); - if (ret != 0) - { - DPRINTF(E_LOG, L_SCAN, "Warning: Could not set thread priority to SCHED_BATCH\n"); - } -#endif - - ret = db_perthread_init(); - if (ret < 0) - { - DPRINTF(E_LOG, L_SCAN, "Error: DB init failed\n"); - - pthread_exit(NULL); - } - - ret = db_watch_clear(); - if (ret < 0) - { - DPRINTF(E_LOG, L_SCAN, "Error: could not clear old watches from DB\n"); - - pthread_exit(NULL); - } - - // Only clear the queue if enabled (default) in config - clear_queue_on_stop_disabled = cfg_getbool(cfg_getsec(cfg, "mpd"), "clear_queue_on_stop_disable"); - if (!clear_queue_on_stop_disabled) - { - ret = db_queue_clear(); - if (ret < 0) - { - DPRINTF(E_LOG, L_SCAN, "Error: could not clear queue from DB\n"); - - pthread_exit(NULL); - } - } - - if (cfg_getbool(cfg_getsec(cfg, "library"), "filescan_disable")) - bulk_scan(F_SCAN_BULK | F_SCAN_FAST); - else - bulk_scan(F_SCAN_BULK); - - if (!scan_exit) - { -#ifdef HAVE_SPOTIFY_H - spotify_login(NULL); -#endif - - /* Enable inotify */ - event_add(inoev, NULL); - - event_base_dispatch(evbase_scan); - } - - if (!scan_exit) - DPRINTF(E_FATAL, L_SCAN, "Scan event loop terminated ahead of time!\n"); - - db_perthread_deinit(); - - pthread_exit(NULL); } static int @@ -1904,10 +1421,10 @@ inofd_event_set(void) return -1; } - inoev = event_new(evbase_scan, inofd, EV_READ, inotify_cb, NULL); + inoev = event_new(evbase_lib, inofd, EV_READ, inotify_cb, NULL); #ifndef __linux__ - deferred_inoev = evtimer_new(evbase_scan, inotify_deferred_cb, NULL); + deferred_inoev = evtimer_new(evbase_lib, inotify_deferred_cb, NULL); if (!deferred_inoev) { DPRINTF(E_LOG, L_SCAN, "Could not create deferred inotify event\n"); @@ -1931,9 +1448,33 @@ inofd_event_unset(void) } /* Thread: scan */ +static int +filescanner_initscan() +{ + int ret; -static enum command_state -filescanner_initscan(void *arg, int *retval) + ret = db_watch_clear(); + if (ret < 0) + { + DPRINTF(E_LOG, L_SCAN, "Error: could not clear old watches from DB\n"); + return -1; + } + + if (cfg_getbool(cfg_getsec(cfg, "library"), "filescan_disable")) + bulk_scan(F_SCAN_BULK | F_SCAN_FAST); + else + bulk_scan(F_SCAN_BULK); + + if (!library_is_exiting()) + { + /* Enable inotify */ + event_add(inoev, NULL); + } + return 0; +} + +static int +filescanner_rescan() { DPRINTF(E_LOG, L_SCAN, "Startup rescan triggered\n"); @@ -1943,12 +1484,11 @@ filescanner_initscan(void *arg, int *retval) inofd_event_set(); bulk_scan(F_SCAN_BULK | F_SCAN_RESCAN); - *retval = 0; - return COMMAND_END; + return 0; } -static enum command_state -filescanner_fullrescan(void *arg, int *retval) +static int +filescanner_fullrescan() { DPRINTF(E_LOG, L_SCAN, "Full rescan triggered\n"); @@ -1960,112 +1500,35 @@ filescanner_fullrescan(void *arg, int *retval) inofd_event_set(); bulk_scan(F_SCAN_BULK); - *retval = 0; - return COMMAND_END; -} - -void -filescanner_trigger_initscan(void) -{ - if (scanning) - { - DPRINTF(E_INFO, L_SCAN, "Scan already running, ignoring request to trigger a new init scan\n"); - return; - } - - commands_exec_async(cmdbase, filescanner_initscan, NULL); -} - -void -filescanner_trigger_fullrescan(void) -{ - if (scanning) - { - DPRINTF(E_INFO, L_SCAN, "Scan already running, ignoring request to trigger a new init scan\n"); - return; - } - - commands_exec_async(cmdbase, filescanner_fullrescan, NULL); -} - -/* - * Query the status of the filescanner - * @return 1 if scan is running, otherwise 0 - */ -int -filescanner_scanning(void) -{ - return scanning; + return 0; } /* Thread: main */ -int +static int filescanner_init(void) { int ret; - scan_exit = 0; - scanning = 0; - - evbase_scan = event_base_new(); - if (!evbase_scan) - { - DPRINTF(E_FATAL, L_SCAN, "Could not create an event base\n"); - - return -1; - } - ret = inofd_event_set(); - if (ret < 0) - { - goto ino_fail; - } - cmdbase = commands_base_new(evbase_scan, NULL); - - ret = pthread_create(&tid_scan, NULL, filescanner, NULL); - if (ret != 0) - { - DPRINTF(E_FATAL, L_SCAN, "Could not spawn filescanner thread: %s\n", strerror(errno)); - - goto thread_fail; - } - -#if defined(HAVE_PTHREAD_SETNAME_NP) - pthread_setname_np(tid_scan, "filescanner"); -#elif defined(HAVE_PTHREAD_SET_NAME_NP) - pthread_set_name_np(tid_scan, "filescanner"); -#endif - - return 0; - - thread_fail: - commands_base_free(cmdbase); - close(inofd); - ino_fail: - event_base_free(evbase_scan); - - return -1; + return ret; } /* Thread: main */ -void +static void filescanner_deinit(void) { - int ret; - - scan_exit = 1; - commands_base_destroy(cmdbase); - - ret = pthread_join(tid_scan, NULL); - if (ret != 0) - { - DPRINTF(E_FATAL, L_SCAN, "Could not join filescanner thread: %s\n", strerror(errno)); - - return; - } - inofd_event_unset(); - - event_base_free(evbase_scan); } + + +struct library_source filescanner = +{ + .name = "filescanner", + .disabled = 0, + .init = filescanner_init, + .deinit = filescanner_deinit, + .initscan = filescanner_initscan, + .rescan = filescanner_rescan, + .fullrescan = filescanner_fullrescan, +}; diff --git a/src/filescanner.h b/src/filescanner.h index 915f99b0..2a858f99 100644 --- a/src/filescanner.h +++ b/src/filescanner.h @@ -12,19 +12,8 @@ #define F_SCAN_TYPE_SPOTIFY (1 << 5) #define F_SCAN_TYPE_PIPE (1 << 6) -int -filescanner_init(void); - -void -filescanner_deinit(void); - -void -filescanner_process_media(char *path, time_t mtime, off_t size, int type, struct media_file_info *external_mfi, int dir_id); /* Actual scanners */ -int -scan_metadata_ffmpeg(char *file, struct media_file_info *mfi); - int scan_metadata_icy(char *url, struct media_file_info *mfi); @@ -39,13 +28,4 @@ void scan_itunes_itml(char *file); #endif -void -filescanner_trigger_initscan(void); - -void -filescanner_trigger_fullrescan(void); - -int -filescanner_scanning(void); - #endif /* !__FILESCANNER_H__ */ diff --git a/src/filescanner_ffmpeg.c b/src/filescanner_ffmpeg.c index 847bf453..2528dd55 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/filescanner_ffmpeg.c @@ -351,7 +351,7 @@ extract_metadata(struct media_file_info *mfi, AVFormatContext *ctx, AVStream *au } int -scan_metadata_ffmpeg(char *file, struct media_file_info *mfi) +scan_metadata_ffmpeg(const char *file, struct media_file_info *mfi) { AVFormatContext *ctx; AVDictionary *options; diff --git a/src/filescanner_playlist.c b/src/filescanner_playlist.c index d393e32b..dffcd228 100644 --- a/src/filescanner_playlist.c +++ b/src/filescanner_playlist.c @@ -38,6 +38,7 @@ #include "db.h" #include "filescanner.h" #include "misc.h" +#include "library.h" /* Formats we can read so far */ #define PLAYLIST_PLS 1 @@ -238,7 +239,7 @@ scan_playlist(char *file, time_t mtime, int dir_id) if (extinf) DPRINTF(E_INFO, L_SCAN, "Playlist has EXTINF metadata, artist is '%s', title is '%s'\n", mfi.artist, mfi.title); - filescanner_process_media(filename, mtime, 0, F_SCAN_TYPE_URL, &mfi, DIR_HTTP); + library_process_media(filename, mtime, 0, DATA_KIND_HTTP, 0, false, &mfi, DIR_HTTP); } /* Regular file, should already be in library */ else diff --git a/src/library.c b/src/library.c new file mode 100644 index 00000000..b3ab3592 --- /dev/null +++ b/src/library.c @@ -0,0 +1,782 @@ +/* + * Copyright (C) 2015 Christian Meffert + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "library.h" + +#include +#include +#include +#include +#include +#ifdef HAVE_PTHREAD_NP_H +# include +#endif +#include +#include +#include +#include +#include + +#include "commands.h" +#include "conffile.h" +#include "db.h" +#include "logger.h" +#include "misc.h" + + +static struct commands_base *cmdbase; +static pthread_t tid_library; + +struct event_base *evbase_lib; + +/* Flag for aborting scan on exit */ +static bool scan_exit; + +/* Flag for scan in progress */ +static bool scanning; + +extern struct library_source filescanner; +#ifdef HAVE_SPOTIFY_H +extern struct library_source spotifyscanner; +#endif + +static struct library_source *sources[] = { + &filescanner, +#ifdef HAVE_SPOTIFY_H + &spotifyscanner, +#endif + NULL +}; + + +static void +sort_tag_create(char **sort_tag, char *src_tag) +{ + const uint8_t *i_ptr; + const uint8_t *n_ptr; + const uint8_t *number; + uint8_t out[1024]; + uint8_t *o_ptr; + int append_number; + ucs4_t puc; + int numlen; + size_t len; + int charlen; + + /* Note: include terminating NUL in string length for u8_normalize */ + + if (*sort_tag) + { + DPRINTF(E_DBG, L_SCAN, "Existing sort tag will be normalized: %s\n", *sort_tag); + o_ptr = u8_normalize(UNINORM_NFD, (uint8_t *)*sort_tag, strlen(*sort_tag) + 1, NULL, &len); + free(*sort_tag); + *sort_tag = (char *)o_ptr; + return; + } + + if (!src_tag || ((len = strlen(src_tag)) == 0)) + { + *sort_tag = NULL; + return; + } + + // Set input pointer past article if present + if ((strncasecmp(src_tag, "a ", 2) == 0) && (len > 2)) + i_ptr = (uint8_t *)(src_tag + 2); + else if ((strncasecmp(src_tag, "an ", 3) == 0) && (len > 3)) + i_ptr = (uint8_t *)(src_tag + 3); + else if ((strncasecmp(src_tag, "the ", 4) == 0) && (len > 4)) + i_ptr = (uint8_t *)(src_tag + 4); + else + i_ptr = (uint8_t *)src_tag; + + // Poor man's natural sort. Makes sure we sort like this: a1, a2, a10, a11, a21, a111 + // We do this by padding zeroes to (short) numbers. As an alternative we could have + // made a proper natural sort algorithm in sqlext.c, but we don't, since we don't + // want any risk of hurting response times + memset(&out, 0, sizeof(out)); + o_ptr = (uint8_t *)&out; + number = NULL; + append_number = 0; + + do + { + n_ptr = u8_next(&puc, i_ptr); + + if (uc_is_digit(puc)) + { + if (!number) // We have encountered the beginning of a number + number = i_ptr; + append_number = (n_ptr == NULL); // If last char in string append number now + } + else + { + if (number) + append_number = 1; // A number has ended so time to append it + else + { + charlen = u8_strmblen(i_ptr); + if (charlen >= 0) + o_ptr = u8_stpncpy(o_ptr, i_ptr, charlen); // No numbers in sight, just append char + } + } + + // Break if less than 100 bytes remain (prevent buffer overflow) + if (sizeof(out) - u8_strlen(out) < 100) + break; + + // Break if number is very large (prevent buffer overflow) + if (number && (i_ptr - number > 50)) + break; + + if (append_number) + { + numlen = i_ptr - number; + if (numlen < 5) // Max pad width + { + u8_strcpy(o_ptr, (uint8_t *)"00000"); + o_ptr += (5 - numlen); + } + o_ptr = u8_stpncpy(o_ptr, number, numlen + u8_strmblen(i_ptr)); + + number = NULL; + append_number = 0; + } + + i_ptr = n_ptr; + } + while (n_ptr); + + *sort_tag = (char *)u8_normalize(UNINORM_NFD, (uint8_t *)&out, u8_strlen(out) + 1, NULL, &len); +} + +static void +fixup_tags(struct media_file_info *mfi) +{ + cfg_t *lib; + size_t len; + char *tag; + char *sep = " - "; + char *ca; + + if (mfi->genre && (strlen(mfi->genre) == 0)) + { + free(mfi->genre); + mfi->genre = NULL; + } + + if (mfi->artist && (strlen(mfi->artist) == 0)) + { + free(mfi->artist); + mfi->artist = NULL; + } + + if (mfi->title && (strlen(mfi->title) == 0)) + { + free(mfi->title); + mfi->title = NULL; + } + + /* + * Default to mpeg4 video/audio for unknown file types + * in an attempt to allow streaming of DRM-afflicted files + */ + if (mfi->codectype && strcmp(mfi->codectype, "unkn") == 0) + { + if (mfi->has_video) + { + strcpy(mfi->codectype, "mp4v"); + strcpy(mfi->type, "m4v"); + } + else + { + strcpy(mfi->codectype, "mp4a"); + strcpy(mfi->type, "m4a"); + } + } + + if (!mfi->artist) + { + if (mfi->orchestra && mfi->conductor) + { + len = strlen(mfi->orchestra) + strlen(sep) + strlen(mfi->conductor); + tag = (char *)malloc(len + 1); + if (tag) + { + sprintf(tag,"%s%s%s", mfi->orchestra, sep, mfi->conductor); + mfi->artist = tag; + } + } + else if (mfi->orchestra) + { + mfi->artist = strdup(mfi->orchestra); + } + else if (mfi->conductor) + { + mfi->artist = strdup(mfi->conductor); + } + } + + /* Handle TV shows, try to present prettier metadata */ + if (mfi->tv_series_name && strlen(mfi->tv_series_name) != 0) + { + mfi->media_kind = MEDIA_KIND_TVSHOW; /* tv show */ + + /* Default to artist = series_name */ + if (mfi->artist && strlen(mfi->artist) == 0) + { + free(mfi->artist); + mfi->artist = NULL; + } + + if (!mfi->artist) + mfi->artist = strdup(mfi->tv_series_name); + + /* Default to album = ", Season " */ + if (mfi->album && strlen(mfi->album) == 0) + { + free(mfi->album); + mfi->album = NULL; + } + + if (!mfi->album) + { + len = snprintf(NULL, 0, "%s, Season %d", mfi->tv_series_name, mfi->tv_season_num); + + mfi->album = (char *)malloc(len + 1); + if (mfi->album) + sprintf(mfi->album, "%s, Season %d", mfi->tv_series_name, mfi->tv_season_num); + } + } + + /* Check the 4 top-tags are filled */ + if (!mfi->artist) + mfi->artist = strdup("Unknown artist"); + if (!mfi->album) + mfi->album = strdup("Unknown album"); + if (!mfi->genre) + mfi->genre = strdup("Unknown genre"); + if (!mfi->title) + { + /* fname is left untouched by unicode_fixup_mfi() for + * obvious reasons, so ensure it is proper UTF-8 + */ + mfi->title = unicode_fixup_string(mfi->fname, "ascii"); + if (mfi->title == mfi->fname) + mfi->title = strdup(mfi->fname); + } + + /* Ensure sort tags are filled, manipulated and normalized */ + sort_tag_create(&mfi->artist_sort, mfi->artist); + sort_tag_create(&mfi->album_sort, mfi->album); + sort_tag_create(&mfi->title_sort, mfi->title); + + /* We need to set album_artist according to media type and config */ + if (mfi->compilation) /* Compilation */ + { + lib = cfg_getsec(cfg, "library"); + ca = cfg_getstr(lib, "compilation_artist"); + if (ca && mfi->album_artist) + { + free(mfi->album_artist); + mfi->album_artist = strdup(ca); + } + else if (ca && !mfi->album_artist) + { + mfi->album_artist = strdup(ca); + } + else if (!ca && !mfi->album_artist) + { + mfi->album_artist = strdup(""); + mfi->album_artist_sort = strdup(""); + } + } + else if (mfi->media_kind == MEDIA_KIND_PODCAST) /* Podcast */ + { + if (mfi->album_artist) + free(mfi->album_artist); + mfi->album_artist = strdup(""); + mfi->album_artist_sort = strdup(""); + } + else if (!mfi->album_artist) /* Regular media without album_artist */ + { + mfi->album_artist = strdup(mfi->artist); + } + + if (!mfi->album_artist_sort && (strcmp(mfi->album_artist, mfi->artist) == 0)) + mfi->album_artist_sort = strdup(mfi->artist_sort); + else + sort_tag_create(&mfi->album_artist_sort, mfi->album_artist); + + /* Composer is not one of our mandatory tags, so take extra care */ + if (mfi->composer_sort || mfi->composer) + sort_tag_create(&mfi->composer_sort, mfi->composer); +} + +void +library_process_media(const char *path, time_t mtime, off_t size, enum data_kind data_kind, enum media_kind force_media_kind, bool force_compilation, struct media_file_info *external_mfi, int dir_id) +{ + struct media_file_info *mfi; + const char *filename; + time_t stamp; + int id; + char virtual_path[PATH_MAX]; + int ret; + + filename = strrchr(path, '/'); + if ((!filename) || (strlen(filename) == 1)) + filename = path; + else + filename++; + + db_file_stamp_bypath(path, &stamp, &id); + + if (stamp && (stamp >= mtime)) + { + db_file_ping(id); + return; + } + + if (!external_mfi) + { + mfi = (struct media_file_info*)malloc(sizeof(struct media_file_info)); + if (!mfi) + { + DPRINTF(E_LOG, L_SCAN, "Out of memory for mfi\n"); + return; + } + + memset(mfi, 0, sizeof(struct media_file_info)); + } + else + mfi = external_mfi; + + if (stamp) + mfi->id = id; + + mfi->fname = strdup(filename); + if (!mfi->fname) + { + DPRINTF(E_LOG, L_SCAN, "Out of memory for fname\n"); + goto out; + } + + mfi->path = strdup(path); + if (!mfi->path) + { + DPRINTF(E_LOG, L_SCAN, "Out of memory for path\n"); + goto out; + } + + mfi->time_modified = mtime; + mfi->file_size = size; + + if (force_compilation) + mfi->compilation = 1; + if (force_media_kind) + mfi->media_kind = force_media_kind; + + if (data_kind == DATA_KIND_FILE) + { + mfi->data_kind = DATA_KIND_FILE; + ret = scan_metadata_ffmpeg(path, mfi); + } + else if (data_kind == DATA_KIND_HTTP) + { + mfi->data_kind = DATA_KIND_HTTP; + ret = scan_metadata_ffmpeg(path, mfi); + if (ret < 0) + { + DPRINTF(E_LOG, L_SCAN, "Playlist URL '%s' is unavailable for probe/metadata, assuming MP3 encoding\n", path); + mfi->type = strdup("mp3"); + mfi->codectype = strdup("mpeg"); + mfi->description = strdup("MPEG audio file"); + ret = 1; + } + } + else if (data_kind == DATA_KIND_SPOTIFY) + { + mfi->data_kind = DATA_KIND_SPOTIFY; + ret = mfi->artist && mfi->album && mfi->title; + } + else if (data_kind == DATA_KIND_PIPE) + { + mfi->data_kind = DATA_KIND_PIPE; + mfi->type = strdup("wav"); + mfi->codectype = strdup("wav"); + mfi->description = strdup("PCM16 pipe"); + ret = 1; + } + else + { + DPRINTF(E_LOG, L_SCAN, "Unknown scan type for '%s', this error should not occur\n", path); + ret = -1; + } + + if (ret < 0) + { + DPRINTF(E_INFO, L_SCAN, "Could not extract metadata for '%s'\n", path); + goto out; + } + + if (!mfi->item_kind) + mfi->item_kind = 2; /* music */ + if (!mfi->media_kind) + mfi->media_kind = MEDIA_KIND_MUSIC; /* music */ + + unicode_fixup_mfi(mfi); + + fixup_tags(mfi); + + if (data_kind == DATA_KIND_HTTP) + { + snprintf(virtual_path, PATH_MAX, "/http:/%s", mfi->title); + mfi->virtual_path = strdup(virtual_path); + } + else if (data_kind == DATA_KIND_SPOTIFY) + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s/%s/%s", mfi->album_artist, mfi->album, mfi->title); + mfi->virtual_path = strdup(virtual_path); + } + else + { + snprintf(virtual_path, PATH_MAX, "/file:%s", mfi->path); + mfi->virtual_path = strdup(virtual_path); + } + + mfi->directory_id = dir_id; + + if (mfi->id == 0) + db_file_add(mfi); + else + db_file_update(mfi); + + out: + if (!external_mfi) + free_mfi(mfi, 0); +} + +int +library_add_playlist_info(const char *path, const char *title, const char *virtual_path, enum pl_type type, int parent_pl_id, int dir_id) +{ + struct playlist_info *pli; + int plid; + int ret; + + pli = db_pl_fetch_bypath(path); + if (pli) + { + DPRINTF(E_DBG, L_LIB, "Playlist found ('%s', link %s), updating\n", title, path); + + plid = pli->id; + + pli->type = type; + free(pli->title); + pli->title = strdup(title); + if (pli->virtual_path) + free(pli->virtual_path); + pli->virtual_path = safe_strdup(virtual_path); + pli->directory_id = dir_id; + + ret = db_pl_update(pli); + if (ret < 0) + { + DPRINTF(E_LOG, L_LIB, "Error updating playlist ('%s', link %s)\n", title, path); + + free_pli(pli, 0); + return -1; + } + + db_pl_clear_items(plid); + } + else + { + DPRINTF(E_DBG, L_LIB, "Adding playlist ('%s', link %s)\n", title, path); + + pli = (struct playlist_info *)malloc(sizeof(struct playlist_info)); + if (!pli) + { + DPRINTF(E_LOG, L_LIB, "Out of memory\n"); + + return -1; + } + + memset(pli, 0, sizeof(struct playlist_info)); + + pli->type = type; + pli->title = strdup(title); + pli->path = strdup(path); + pli->virtual_path = safe_strdup(virtual_path); + pli->parent_id = parent_pl_id; + pli->directory_id = dir_id; + + ret = db_pl_add(pli, &plid); + if ((ret < 0) || (plid < 1)) + { + DPRINTF(E_LOG, L_LIB, "Error adding playlist ('%s', link %s, ret %d, plid %d)\n", title, path, ret, plid); + + free_pli(pli, 0); + return -1; + } + } + + free_pli(pli, 0); + return plid; +} + +static enum command_state +rescan(void *arg, int *ret) +{ + time_t starttime; + time_t endtime; + int i; + + starttime = time(NULL); + + for (i = 0; sources[i]; i++) + { + if (!sources[i]->disabled && sources[i]->rescan) + sources[i]->rescan(); + } + + endtime = time(NULL); + DPRINTF(E_LOG, L_LIB, "Library rescan completed in %.f sec\n", difftime(endtime, starttime)); + + scanning = false; + *ret = 0; + return COMMAND_END; +} + +static enum command_state +fullrescan(void *arg, int *ret) +{ + int i; + + for (i = 0; sources[i]; i++) + { + if (!sources[i]->disabled && sources[i]->fullrescan) + sources[i]->fullrescan(); + } + + scanning = false; + *ret = 0; + return COMMAND_END; +} + +void +library_rescan() +{ + if (scanning) + { + DPRINTF(E_INFO, L_LIB, "Scan already running, ignoring request to trigger a new init scan\n"); + return; + } + + scanning = true; // TODO Guard "scanning" with a mutex + commands_exec_async(cmdbase, rescan, NULL); +} + +void +library_fullrescan() +{ + if (scanning) + { + DPRINTF(E_INFO, L_LIB, "Scan already running, ignoring request to trigger a new full rescan\n"); + return; + } + + scanning = true; // TODO Guard "scanning" with a mutex + commands_exec_async(cmdbase, fullrescan, NULL); +} + +static void +initscan() +{ + time_t starttime; + time_t endtime; + bool clear_queue_disabled; + int i; + + scanning = true; + starttime = time(NULL); + + // Only clear the queue if enabled (default) in config + clear_queue_disabled = cfg_getbool(cfg_getsec(cfg, "mpd"), "clear_queue_on_stop_disable"); + if (!clear_queue_disabled) + { + db_queue_clear(); + } + + for (i = 0; sources[i]; i++) + { + if (!sources[i]->disabled && sources[i]->initscan) + sources[i]->initscan(); + } + + endtime = time(NULL); + DPRINTF(E_LOG, L_LIB, "Library init scan completed in %.f sec\n", difftime(endtime, starttime)); + + scanning = false; +} + +/* + * @return true if scan is running, otherwise false + */ +bool +library_is_scanning() +{ + return scanning; +} + +/* + * @param is_scanning true if scan is running, otherwise false + */ +void +library_set_scanning(bool is_scanning) +{ + scanning = is_scanning; +} + +/* + * @return true if a running scan should be aborted due to imminent shutdown, otherwise false + */ +bool +library_is_exiting() +{ + return scan_exit; +} + +static void * +library(void *arg) +{ + int ret; + +#ifdef __linux__ + struct sched_param param; + + /* Lower the priority of the thread so forked-daapd may still respond + * during library scan on low power devices. Param must be 0 for the SCHED_BATCH + * policy. + */ + memset(¶m, 0, sizeof(struct sched_param)); + ret = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (ret != 0) + { + DPRINTF(E_LOG, L_LIB, "Warning: Could not set thread priority to SCHED_BATCH\n"); + } +#endif + + ret = db_perthread_init(); + if (ret < 0) + { + DPRINTF(E_LOG, L_LIB, "Error: DB init failed\n"); + + pthread_exit(NULL); + } + + initscan(); + + event_base_dispatch(evbase_lib); + + if (!scan_exit) + DPRINTF(E_FATAL, L_LIB, "Scan event loop terminated ahead of time!\n"); + + db_perthread_deinit(); + + pthread_exit(NULL); +} + +/* Thread: main */ +int +library_init(void) +{ + int i; + int ret; + + scan_exit = false; + scanning = false; + + evbase_lib = event_base_new(); + if (!evbase_lib) + { + DPRINTF(E_FATAL, L_LIB, "Could not create an event base\n"); + + return -1; + } + + for (i = 0; sources[i]; i++) + { + ret = sources[i]->init(); + if (ret < 0) + sources[i]->disabled = 1; + } + + cmdbase = commands_base_new(evbase_lib, NULL); + + ret = pthread_create(&tid_library, NULL, library, NULL); + if (ret != 0) + { + DPRINTF(E_FATAL, L_LIB, "Could not spawn library thread: %s\n", strerror(errno)); + + goto thread_fail; + } + +#if defined(HAVE_PTHREAD_SETNAME_NP) + pthread_setname_np(tid_library, "library"); +#elif defined(HAVE_PTHREAD_SET_NAME_NP) + pthread_set_name_np(tid_library, "library"); +#endif + + return 0; + + thread_fail: + event_base_free(evbase_lib); + + return -1; +} + +/* Thread: main */ +void +library_deinit() +{ + int i; + int ret; + + scan_exit = true; + commands_base_destroy(cmdbase); + + ret = pthread_join(tid_library, NULL); + if (ret != 0) + { + DPRINTF(E_FATAL, L_LIB, "Could not join library thread: %s\n", strerror(errno)); + + return; + } + + for (i = 0; sources[i]; i++) + { + sources[i]->deinit(); + } + + event_base_free(evbase_lib); +} diff --git a/src/library.h b/src/library.h new file mode 100644 index 00000000..b703fbdf --- /dev/null +++ b/src/library.h @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2015 Christian Meffert + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef SRC_LIBRARY_H_ +#define SRC_LIBRARY_H_ + +#include +#include +#include + +#include "db.h" + +/* + * Definition of a library source + * + * A library source is responsible for scanning items into the library db. + */ +struct library_source +{ + char *name; + int disabled; + + /* + * Initialize library source (called from the main thread) + */ + int (*init)(void); + + /* + * Shutdown library source (called from the main thread after + * terminating the library thread) + */ + void (*deinit)(void); + + /* + * Run initial scan after startup (called from the library thread) + */ + int (*initscan)(void); + + /* + * Run rescan (called from the library thread) + */ + int (*rescan)(void); + + /* + * Run a full rescan (purge library entries and rescan) (called from the library thread) + */ + int (*fullrescan)(void); + +}; + +int +scan_metadata_ffmpeg(const char *file, struct media_file_info *mfi); + + +void +library_process_media(const char *path, time_t mtime, off_t size, enum data_kind data_kind, enum media_kind force_media_kind, bool force_compilation, struct media_file_info *external_mfi, int dir_id); + +int +library_add_playlist_info(const char *path, const char *title, const char *virtual_path, enum pl_type type, int parent_pl_id, int dir_id); + +void +library_rescan(); + +void +library_fullrescan(); + +bool +library_is_scanning(); + +void +library_set_scanning(bool is_scanning); + +bool +library_is_exiting(); + +int +library_init(); + +void +library_deinit(); + + +#endif /* SRC_LIBRARY_H_ */ diff --git a/src/logger.c b/src/logger.c index fa216e2a..3da8ceef 100644 --- a/src/logger.c +++ b/src/logger.c @@ -43,7 +43,7 @@ static int threshold; static int console; static char *logfilename; static FILE *logfile; -static char *labels[] = { "config", "daap", "db", "httpd", "http", "main", "mdns", "misc", "rsp", "scan", "xcode", "event", "remote", "dacp", "ffmpeg", "artwork", "player", "raop", "laudio", "dmap", "dbperf", "spotify", "lastfm", "cache", "mpd", "stream", "cast", "fifo" }; +static char *labels[] = { "config", "daap", "db", "httpd", "http", "main", "mdns", "misc", "rsp", "scan", "xcode", "event", "remote", "dacp", "ffmpeg", "artwork", "player", "raop", "laudio", "dmap", "dbperf", "spotify", "lastfm", "cache", "mpd", "stream", "cast", "fifo", "lib" }; static char *severities[] = { "FATAL", "LOG", "WARN", "INFO", "DEBUG", "SPAM" }; diff --git a/src/logger.h b/src/logger.h index e506edf0..dff5258a 100644 --- a/src/logger.h +++ b/src/logger.h @@ -34,8 +34,9 @@ #define L_STREAMING 25 #define L_CAST 26 #define L_FIFO 27 +#define L_LIB 28 -#define N_LOGDOMAINS 28 +#define N_LOGDOMAINS 29 /* Severities */ #define E_FATAL 0 diff --git a/src/main.c b/src/main.c index ef7eb84e..7041b7cc 100644 --- a/src/main.c +++ b/src/main.c @@ -60,20 +60,17 @@ GCRY_THREAD_OPTION_PTHREAD_IMPL; #include "logger.h" #include "misc.h" #include "cache.h" -#include "filescanner.h" #include "httpd.h" #include "mpd.h" #include "mdns.h" #include "remote_pairing.h" #include "player.h" #include "worker.h" +#include "library.h" #ifdef HAVE_LIBCURL # include #endif -#ifdef HAVE_SPOTIFY_H -# include "spotify.h" -#endif #define PIDFILE STATEDIR "/run/" PACKAGE ".pid" @@ -738,25 +735,16 @@ main(int argc, char **argv) goto cache_fail; } - /* Spawn file scanner thread */ - ret = filescanner_init(); + /* Spawn library scan thread */ + ret = library_init(); if (ret != 0) { - DPRINTF(E_FATAL, L_MAIN, "File scanner thread failed to start\n"); + DPRINTF(E_FATAL, L_MAIN, "Library thread failed to start\n"); ret = EXIT_FAILURE; - goto filescanner_fail; + goto library_fail; } -#ifdef HAVE_SPOTIFY_H - /* Spawn Spotify thread */ - ret = spotify_init(); - if (ret < 0) - { - DPRINTF(E_INFO, L_MAIN, "Spotify thread not started\n");; - } -#endif - /* Spawn player thread */ ret = player_init(); if (ret != 0) @@ -895,14 +883,10 @@ main(int argc, char **argv) player_deinit(); player_fail: -#ifdef HAVE_SPOTIFY_H - DPRINTF(E_LOG, L_MAIN, "Spotify deinit\n"); - spotify_deinit(); -#endif - DPRINTF(E_LOG, L_MAIN, "File scanner deinit\n"); - filescanner_deinit(); + DPRINTF(E_LOG, L_MAIN, "Library scaner deinit\n"); + library_deinit(); - filescanner_fail: + library_fail: DPRINTF(E_LOG, L_MAIN, "Cache deinit\n"); cache_deinit(); diff --git a/src/mpd.c b/src/mpd.c index 05c0db45..ad6242d8 100644 --- a/src/mpd.c +++ b/src/mpd.c @@ -56,8 +56,8 @@ #include "artwork.h" #include "player.h" -#include "filescanner.h" #include "commands.h" +#include "library.h" static pthread_t tid_mpd; @@ -748,7 +748,7 @@ mpd_command_status(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) (status.pos_ms / 1000.0)); } - if (filescanner_scanning()) + if (library_is_scanning()) { evbuffer_add(evbuf, "updating_db: 1\n", 15); } @@ -3218,7 +3218,7 @@ mpd_command_update(struct evbuffer *evbuf, int argc, char **argv, char **errmsg) return ACK_ERROR_ARG; } - filescanner_trigger_initscan(); + library_rescan(); evbuffer_add(evbuf, "updating_db: 1\n", 15); diff --git a/src/spotify.c b/src/spotify.c index 53ea8805..f040d076 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -50,9 +51,9 @@ #include "misc.h" #include "http.h" #include "conffile.h" -#include "filescanner.h" #include "cache.h" #include "commands.h" +#include "library.h" /* TODO for the web api: * - UI should be prettier @@ -711,7 +712,7 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde // DPRINTF(E_DBG, L_SPOTIFY, "Saving track '%s': '%s' by %s (%s)\n", url, mfi.title, mfi.artist, mfi.album); - filescanner_process_media(url, time(NULL), 0, F_SCAN_TYPE_SPOTIFY, &mfi, dir_id); + library_process_media(url, time(NULL), 0, DATA_KIND_SPOTIFY, 0, 0, &mfi, dir_id); free_mfi(&mfi, 1); @@ -2554,6 +2555,28 @@ spotify_login(char *path) } } +/* Thread: library */ +static int +spotify_initscan() +{ + spotify_login(NULL); + return 0; +} + +/* Thread: library */ +static int +spotify_rescan() +{ + return 0; +} + +/* Thread: library */ +static int +spotify_fullrescan() +{ + return 0; +} + /* Thread: main */ int spotify_init(void) @@ -2748,3 +2771,14 @@ spotify_deinit(void) /* Release libspotify handle */ dlclose(g_libhandle); } + +struct library_source spotifyscanner = +{ + .name = "spotifyscanner", + .disabled = 0, + .init = spotify_init, + .deinit = spotify_deinit, + .rescan = spotify_rescan, + .initscan = spotify_initscan, + .fullrescan = spotify_fullrescan, +}; From 5efadb6fc29527319354cb904a9ad51da940a798 Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 31 Dec 2016 16:46:59 +0100 Subject: [PATCH 02/33] [spotify_webapi] Add separate file for accessing the web api --- src/Makefile.am | 2 +- src/spotify_webapi.c | 725 +++++++++++++++++++++++++++++++++++++++++++ src/spotify_webapi.h | 124 ++++++++ 3 files changed, 850 insertions(+), 1 deletion(-) create mode 100644 src/spotify_webapi.c create mode 100644 src/spotify_webapi.h diff --git a/src/Makefile.am b/src/Makefile.am index 71d21290..08a26531 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,7 +6,7 @@ ITUNES_SRC=filescanner_itunes.c endif if COND_SPOTIFY -SPOTIFY_SRC=spotify.c spotify.h +SPOTIFY_SRC=spotify.c spotify.h spotify_webapi.c spotify_webapi.h endif if COND_LASTFM diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c new file mode 100644 index 00000000..7184ffcc --- /dev/null +++ b/src/spotify_webapi.c @@ -0,0 +1,725 @@ +/* + * Copyright (C) 2016 Espen Jürgensen + * Copyright (C) 2016 Christian Meffert + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "spotify_webapi.h" + +#include +#include +#include +#include +#include + +#ifdef HAVE_JSON_C_OLD +# include +#else +# include +#endif + +#include "db.h" +#include "http.h" +#include "library.h" +#include "logger.h" + + + +// Credentials for the web api +static char *spotify_access_token; +static char *spotify_refresh_token; + +static int32_t expires_in = 3600; +static time_t token_requested = 0; + +// Endpoints and credentials for the web api +static const char *spotify_client_id = "0e684a5422384114a8ae7ac020f01789"; +static const char *spotify_client_secret = "232af95f39014c9ba218285a5c11a239"; +static const char *spotify_auth_uri = "https://accounts.spotify.com/authorize"; +static const char *spotify_token_uri = "https://accounts.spotify.com/api/token"; + + +/*--------------------- HELPERS FOR SPOTIFY WEB API -------------------------*/ +/* All the below is in the httpd thread */ + + +static void +jparse_free(json_object* haystack) +{ + if (haystack) + { +#ifdef HAVE_JSON_C_OLD + json_object_put(haystack); +#else + if (json_object_put(haystack) != 1) + DPRINTF(E_LOG, L_SPOTIFY, "Memleak: JSON parser did not free object\n"); +#endif + } +} + +static int +jparse_array_from_obj(json_object *haystack, const char *key, json_object **needle) +{ + if (! (json_object_object_get_ex(haystack, key, needle) && json_object_get_type(*needle) == json_type_array) ) + return -1; + else + return 0; +} + +static const char * +jparse_str_from_obj(json_object *haystack, const char *key) +{ + json_object *needle; + + if (json_object_object_get_ex(haystack, key, &needle) && json_object_get_type(needle) == json_type_string) + return json_object_get_string(needle); + else + return NULL; +} + +static int +jparse_int_from_obj(json_object *haystack, const char *key) +{ + json_object *needle; + + if (json_object_object_get_ex(haystack, key, &needle) && json_object_get_type(needle) == json_type_int) + return json_object_get_int(needle); + else + return 0; +} + +static time_t +jparse_time_from_obj(json_object *haystack, const char *key) +{ + const char *tmp; + struct tm tp; + time_t parsed_time; + + memset(&tp, 0, sizeof(struct tm)); + + tmp = jparse_str_from_obj(haystack, key); + if (!tmp) + return 0; + + strptime(tmp, "%Y-%m-%dT%H:%M:%SZ", &tp); + parsed_time = mktime(&tp); + if (parsed_time < 0) + return 0; + + return parsed_time; +} + +static char * +jparse_artist_from_obj(json_object *artists) +{ + char artistnames[1024]; + json_object *artist; + int count; + int i; + const char *tmp; + + count = json_object_array_length(artists); + if (count == 0) + { + return NULL; + } + + for (i = 0; i < count; i++) + { + artist = json_object_array_get_idx(artists, i); + tmp = jparse_str_from_obj(artist, "name"); + + if (i == 0) + { + strcpy(artistnames, tmp); + } + else + { + strncat(artistnames, ", ", sizeof(artistnames) - 2); + strncat(artistnames, tmp, sizeof(artistnames) - strlen(artistnames)); + } + } + + return strdup(artistnames); +} + +static void +http_client_ctx_free(struct http_client_ctx *ctx) +{ + if (!ctx) + return; + + if (ctx->input_body) + evbuffer_free(ctx->input_body); + if (ctx->output_headers) + { + keyval_clear(ctx->output_headers); + free(ctx->output_headers); + } + free(ctx); +} + +char * +spotifywebapi_oauth_uri_get(const char *redirect_uri) +{ + struct keyval kv; + char *param; + char *uri; + int uri_len; + int ret; + + uri = NULL; + memset(&kv, 0, sizeof(struct keyval)); + ret = ( (keyval_add(&kv, "client_id", spotify_client_id) == 0) && + (keyval_add(&kv, "response_type", "code") == 0) && + (keyval_add(&kv, "redirect_uri", redirect_uri) == 0) && + (keyval_add(&kv, "scope", "playlist-read-private user-library-read") == 0) && + (keyval_add(&kv, "show_dialog", "false") == 0) ); + if (!ret) + { + DPRINTF(E_LOG, L_SPOTIFY, "Cannot display Spotify oath interface (error adding parameters to keyval)\n"); + goto out_clear_kv; + } + + param = http_form_urlencode(&kv); + if (param) + { + uri_len = strlen(spotify_auth_uri) + strlen(param) + 3; + uri = calloc(uri_len, sizeof(char)); + snprintf(uri, uri_len, "%s/?%s", spotify_auth_uri, param); + + free(param); + } + + out_clear_kv: + keyval_clear(&kv); + + return uri; +} + +static int +tokens_get(struct keyval *kv, const char **err) +{ + struct http_client_ctx ctx; + char *param; + char *body; + json_object *haystack; + const char *tmp; + int ret; + + param = http_form_urlencode(kv); + if (!param) + { + *err = "http_form_uriencode() failed"; + ret = -1; + goto out_clear_kv; + } + + memset(&ctx, 0, sizeof(struct http_client_ctx)); + ctx.url = (char *)spotify_token_uri; + ctx.output_body = param; + ctx.input_body = evbuffer_new(); + + ret = http_client_request(&ctx); + if (ret < 0) + { + *err = "Did not get a reply from Spotify"; + goto out_free_input_body; + } + + // 0-terminate for safety + evbuffer_add(ctx.input_body, "", 1); + + body = (char *)evbuffer_pullup(ctx.input_body, -1); + if (!body || (strlen(body) == 0)) + { + *err = "The reply from Spotify is empty or invalid"; + ret = -1; + goto out_free_input_body; + } + + DPRINTF(E_DBG, L_SPOTIFY, "Token reply: %s\n", body); + + haystack = json_tokener_parse(body); + if (!haystack) + { + *err = "JSON parser returned an error"; + ret = -1; + goto out_free_input_body; + } + + tmp = jparse_str_from_obj(haystack, "access_token"); + if (tmp) + spotify_access_token = strdup(tmp); + + tmp = jparse_str_from_obj(haystack, "refresh_token"); + if (tmp) + spotify_refresh_token = strdup(tmp); + + expires_in = jparse_int_from_obj(haystack, "expires_in"); + if (expires_in == 0) + expires_in = 3600; + + jparse_free(haystack); + + if (!spotify_access_token) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not find access token in reply: %s\n", body); + + *err = "Could not find access token in Spotify reply (see log)"; + ret = -1; + goto out_free_input_body; + } + + token_requested = time(NULL); + + DPRINTF(E_LOG, L_SPOTIFY, "token: '%s'\n", spotify_access_token); + DPRINTF(E_LOG, L_SPOTIFY, "refresh-token: '%s'\n", spotify_refresh_token); + DPRINTF(E_LOG, L_SPOTIFY, "expires in: %d\n", expires_in); + + if (spotify_refresh_token) + db_admin_set("spotify_refresh_token", spotify_refresh_token); + + ret = 0; + + out_free_input_body: + evbuffer_free(ctx.input_body); + free(param); + out_clear_kv: + + return ret; +} + +int +spotifywebapi_token_get(const char *code, const char *redirect_uri, const char **err) +{ + struct keyval kv; + int ret; + + memset(&kv, 0, sizeof(struct keyval)); + ret = ( (keyval_add(&kv, "grant_type", "authorization_code") == 0) && + (keyval_add(&kv, "code", code) == 0) && + (keyval_add(&kv, "client_id", spotify_client_id) == 0) && + (keyval_add(&kv, "client_secret", spotify_client_secret) == 0) && + (keyval_add(&kv, "redirect_uri", redirect_uri) == 0) ); + + if (!ret) + { + *err = "Add parameters to keyval failed"; + ret = -1; + } + else + ret = tokens_get(&kv, err); + + keyval_clear(&kv); + + return ret; +} + +int +spotifywebapi_token_refresh() +{ + struct keyval kv; + char *refresh_token; + const char *err; + int ret; + + if (token_requested && difftime(token_requested, time(NULL)) < expires_in) + { + DPRINTF(E_DBG, L_SPOTIFY, "Spotify token still valid\n"); + return 0; + } + + refresh_token = db_admin_get("spotify_refresh_token"); + if (!refresh_token) + { + DPRINTF(E_LOG, L_SPOTIFY, "No spotify refresh token found\n"); + return -1; + } + + DPRINTF(E_DBG, L_SPOTIFY, "Spotify refresh-token: '%s'\n", refresh_token); + + memset(&kv, 0, sizeof(struct keyval)); + ret = ( (keyval_add(&kv, "grant_type", "refresh_token") == 0) && + (keyval_add(&kv, "client_id", spotify_client_id) == 0) && + (keyval_add(&kv, "client_secret", spotify_client_secret) == 0) && + (keyval_add(&kv, "refresh_token", refresh_token) == 0) ); + if (!ret) + { + DPRINTF(E_LOG, L_SPOTIFY, "Add parameters to keyval failed"); + ret = -1; + } + else + ret = tokens_get(&kv, &err); + + free(refresh_token); + keyval_clear(&kv); + + return ret; +} + +int +spotifywebapi_request_uri(struct spotify_request *request, const char *uri) +{ + char bearer_token[1024]; + int ret; + + memset(request, 0, sizeof(struct spotify_request)); + + if (0 > spotifywebapi_token_refresh()) + { + return -1; + } + + request->ctx = calloc(1, sizeof(struct http_client_ctx)); + request->ctx->output_headers = calloc(1, sizeof(struct keyval)); + request->ctx->input_body = evbuffer_new(); + request->ctx->url = uri; + + snprintf(bearer_token, sizeof(bearer_token), "Bearer %s", spotify_access_token); + if (keyval_add(request->ctx->output_headers, "Authorization", bearer_token) < 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Add bearer_token to keyval failed"); + return -1; + } + + ret = http_client_request(request->ctx); + if (ret < 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed"); + return -1; + } + + // 0-terminate for safety + evbuffer_add(request->ctx->input_body, "", 1); + + request->response_body = (char *) evbuffer_pullup(request->ctx->input_body, -1); + if (!request->response_body || (strlen(request->response_body) == 0)) + { + DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed, response was empty"); + return -1; + } + + request->haystack = json_tokener_parse(request->response_body); + if (!request->haystack) + { + DPRINTF(E_LOG, L_SPOTIFY, "JSON parser returned an error\n"); + return -1; + } + + DPRINTF(E_DBG, L_SPOTIFY, "Got response for '%s'\n", uri); + return 0; +} + +void +spotifywebapi_request_end(struct spotify_request *request) +{ + http_client_ctx_free(request->ctx); + jparse_free(request->haystack); +} + +int +spotifywebapi_request_next(struct spotify_request *request, const char *uri) +{ + char *next_uri; + const char *tmp; + int ret; + + if (request->ctx && !request->next_uri) + { + // Reached end of paging requests, terminate loop + return -1; + } + + if (!request->ctx) + { + // First paging request + next_uri = strdup (uri); + } + else + { + // Next paging request + next_uri = strdup(request->next_uri); + spotifywebapi_request_end(request); + } + + ret = spotifywebapi_request_uri(request, next_uri); + free(next_uri); + + if (ret < 0) + return ret; + + request->total = jparse_int_from_obj(request->haystack, "total"); + tmp = jparse_str_from_obj(request->haystack, "next"); + if (tmp) + request->next_uri = strdup(tmp); + + if (jparse_array_from_obj(request->haystack, "items", &request->items) < 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "No items in reply from Spotify. See:\n%s\n", request->response_body); + return -1; + } + + request->count = json_object_array_length(request->items); + + DPRINTF(E_DBG, L_SPOTIFY, "Got %d items\n", request->count); + return 0; +} + +void +track_metadata(json_object* jsontrack, struct spotify_track* track) +{ + json_object* jsonalbum; + json_object* jsonartists; + + if (json_object_object_get_ex(jsontrack, "album", &jsonalbum)) + { + track->album = jparse_str_from_obj(jsonalbum, "name"); + if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + { + track->album_artist = jparse_artist_from_obj(jsonartists); + } + } + if (json_object_object_get_ex(jsontrack, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + { + track->artist = jparse_artist_from_obj(jsonartists); + } + track->disc_number = jparse_int_from_obj(jsontrack, "disc_number"); + track->duration_ms = jparse_int_from_obj(jsontrack, "duration_ms"); + track->name = jparse_str_from_obj(jsontrack, "name"); + track->track_number = jparse_int_from_obj(jsontrack, "track_number"); + track->uri = jparse_str_from_obj(jsontrack, "uri"); + track->id = jparse_str_from_obj(jsontrack, "id"); +} + +int +spotifywebapi_saved_tracks_fetch(struct spotify_request *request, struct spotify_track *track) +{ + json_object *item; + json_object *jsontrack; + + memset(track, 0, sizeof(struct spotify_track)); + + if (request->index >= request->count) + { + return -1; + } + + item = json_object_array_get_idx(request->items, request->index); + if (!(item && json_object_object_get_ex(item, "track", &jsontrack))) + { + DPRINTF(E_LOG, L_SPOTIFY, "Unexpected JSON: Item %d did not have 'track'->'uri'\n", request->index); + request->index++; + return -1; + } + + track_metadata(jsontrack, track); + track->added_at = jparse_str_from_obj(item, "added_at"); +// if (track->added_at) +// strptime("%Y-%m-%dT%H:%M:%SZ"); + + request->index++; + + return 0; +} + +int +spotifywebapi_track(const char *path, struct spotify_track *track) +{ +// char uri[1024]; +// struct http_client_ctx *ctx; +// char *response_body; +// json_object *haystack; +// +// memset(track, 0, sizeof(struct spotify_track)); +// +// if (strlen(path) < 15) // spotify:track:3RZwWEcQHD0Sy1hN1xNYeh +// return -1; +// +// snprintf(uri, sizeof(uri), "%s%s", spotify_track_uri, (path + 14)); +// ctx = request_uri(uri); +// +// if (!ctx) +// return -1; +// +// response_body = (char *) evbuffer_pullup(ctx->input_body, -1); +// if (!response_body || (strlen(response_body) == 0)) +// { +// DPRINTF(E_LOG, L_SPOTIFY, "Request for track failed, response was empty"); +// http_client_ctx_free(ctx); +// return -1; +// } +// +// haystack = json_tokener_parse(response_body); +// track_metadata(haystack, track); +// +// http_client_ctx_free(ctx); +// jparse_free(haystack); + + return 0; +} + +void +album_metadata(json_object *jsonalbum, struct spotify_album *album) +{ + json_object* jsonartists; + + if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + { + album->artist = jparse_artist_from_obj(jsonartists); + } + album->name = jparse_str_from_obj(jsonalbum, "name"); + album->uri = jparse_str_from_obj(jsonalbum, "uri"); + album->id = jparse_str_from_obj(jsonalbum, "id"); + + album->album_type = jparse_str_from_obj(jsonalbum, "album_type"); + album->genre = jparse_str_from_obj(jsonalbum, "genre"); // FIXME Genre is an array of strings ('genres'), but it seems to be always empty + album->label = jparse_str_from_obj(jsonalbum, "label"); + album->release_date = jparse_str_from_obj(jsonalbum, "release_date"); +} + +int +spotifywebapi_saved_albums_fetch(struct spotify_request *request, json_object **jsontracks, int *track_count, struct spotify_album *album) +{ + json_object *jsonalbum; + json_object *item; + json_object *needle; + + memset(album, 0, sizeof(struct spotify_album)); + *track_count = 0; + + if (request->index >= request->count) + { + return -1; + } + + item = json_object_array_get_idx(request->items, request->index); + if (!(item && json_object_object_get_ex(item, "album", &jsonalbum))) + { + DPRINTF(E_LOG, L_SPOTIFY, "Unexpected JSON: Item %d did not have 'album'->'uri'\n", request->index); + request->index++; + return -1; + } + + album_metadata(jsonalbum, album); + + album->added_at = jparse_str_from_obj(item, "added_at"); + album->mtime = jparse_time_from_obj(item, "added_at"); + + if (json_object_object_get_ex(jsonalbum, "tracks", &needle)) + { + if (jparse_array_from_obj(needle, "items", jsontracks) == 0) + { + *track_count = json_object_array_length(*jsontracks); + } + } + + request->index++; + + return 0; +} + +int +spotifywebapi_album_track_fetch(json_object *jsontracks, int index, struct spotify_track *track) +{ + json_object *jsontrack; + + memset(track, 0, sizeof(struct spotify_track)); + + jsontrack = json_object_array_get_idx(jsontracks, index); + + if (!jsontrack) + { + return -1; + } + + track_metadata(jsontrack, track); + + return 0; +} + +void +playlist_metadata(json_object *jsonplaylist, struct spotify_playlist *playlist) +{ + json_object *needle; + + playlist->name = jparse_str_from_obj(jsonplaylist, "name"); + playlist->uri = jparse_str_from_obj(jsonplaylist, "uri"); + playlist->id = jparse_str_from_obj(jsonplaylist, "id"); + playlist->href = jparse_str_from_obj(jsonplaylist, "href"); + + if (json_object_object_get_ex(jsonplaylist, "owner", &needle)) + { + playlist->owner = jparse_str_from_obj(needle, "id"); + } + + if (json_object_object_get_ex(jsonplaylist, "tracks", &needle)) + { + playlist->tracks_href = jparse_str_from_obj(needle, "href"); + playlist->tracks_count = jparse_int_from_obj(needle, "total"); + } +} + +int +spotifywebapi_playlists_fetch(struct spotify_request *request, struct spotify_playlist *playlist) +{ + json_object *jsonplaylist; + + memset(playlist, 0, sizeof(struct spotify_playlist)); + + if (request->index >= request->count) + { + return -1; + } + + jsonplaylist = json_object_array_get_idx(request->items, request->index); + + playlist_metadata(jsonplaylist, playlist); + + request->index++; + + return 0; +} + +int +spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spotify_track *track) +{ + json_object *item; + json_object *jsontrack; + + memset(track, 0, sizeof(struct spotify_track)); + + if (request->index >= request->count) + { + return -1; + } + + item = json_object_array_get_idx(request->items, request->index); + if (!(item && json_object_object_get_ex(item, "track", &jsontrack))) + { + DPRINTF(E_LOG, L_SPOTIFY, "Unexpected JSON: Item %d did not have 'track'->'uri'\n", request->index); + request->index++; + return -1; + } + + track_metadata(jsontrack, track); + track->added_at = jparse_str_from_obj(item, "added_at"); + track->mtime = jparse_time_from_obj(item, "added_at"); + + request->index++; + + return 0; +} diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h new file mode 100644 index 00000000..455d0758 --- /dev/null +++ b/src/spotify_webapi.h @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2016 Espen Jürgensen + * Copyright (C) 2016 Christian Meffert + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef SRC_SPOTIFY_WEBAPI_H_ +#define SRC_SPOTIFY_WEBAPI_H_ + + +#include + +#include "http.h" + +#ifdef HAVE_JSON_C_OLD +# include +#else +# include +#endif + + +#define SPOTIFY_WEBAPI_SAVED_TRACKS "https://api.spotify.com/v1/me/tracks?limit=50" +#define SPOTIFY_WEBAPI_TRACKS "https://api.spotify.com/v1/tracks/" +#define SPOTIFY_WEBAPI_SAVED_ALBUMS "https://api.spotify.com/v1/me/albums?limit=50" +#define SPOTIFY_WEBAPI_SAVED_PLAYLISTS "https://api.spotify.com/v1/me/playlists?limit=50" + +struct spotify_album +{ + const char *added_at; + time_t mtime; + + const char *album_type; + const char *artist; + const char *genre; + const char *id; + const char *label; + const char *name; + const char *release_date; + const char *uri; +}; + +struct spotify_track +{ + const char *added_at; + time_t mtime; + + const char *album; + const char *album_artist; + const char *artist; + int disc_number; + int duration_ms; + const char *id; + const char *name; + int track_number; + const char *uri; +}; + +struct spotify_playlist +{ + const char *id; + const char *name; + const char *owner; + const char *uri; + + const char *href; + + const char *tracks_href; + int tracks_count; +}; + +struct spotify_request +{ + struct http_client_ctx *ctx; + char *response_body; + json_object *haystack; + json_object *items; + int count; + int total; + char *next_uri; + + int index; +}; + +char * +spotifywebapi_oauth_uri_get(const char *redirect_uri); +int +spotifywebapi_token_get(const char *code, const char *redirect_uri, const char **err); +int +spotifywebapi_token_refresh(); + +int +spotifywebapi_request_uri(struct spotify_request *request, const char *uri); +void +spotifywebapi_request_end(struct spotify_request *request); +int +spotifywebapi_request_next(struct spotify_request *request, const char *uri); +int +spotifywebapi_saved_tracks_fetch(struct spotify_request *request, struct spotify_track *track); +int +spotifywebapi_track(const char *path, struct spotify_track *track); +int +spotifywebapi_saved_albums_fetch(struct spotify_request *request, json_object **jsontracks, int *track_count, struct spotify_album *album); +int +spotifywebapi_album_track_fetch(json_object *jsontracks, int index, struct spotify_track *track); +int +spotifywebapi_playlists_fetch(struct spotify_request *request, struct spotify_playlist* playlist); +int +spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spotify_track *track); + + +#endif /* SRC_SPOTIFY_WEBAPI_H_ */ From 1a85257fb31d42c8481395b7f7ddb6fc522e608b Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 31 Dec 2016 16:56:35 +0100 Subject: [PATCH 03/33] [spotify] Extract function for preparing directories --- src/spotify.c | 68 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index f040d076..ccb94461 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -623,6 +623,48 @@ spotify_metadata_get(sp_track *track, struct media_file_info *mfi, const char *p return 0; } +/* + * Returns the directory id for /spotify://, if the directory (or the parent + * directories) does not yet exist, they will be created. + * If an error occured the return value is -1. + * + * @return directory id for the given artist/album directory + */ +static int +prepare_directories(const char *artist, const char *album) +{ + int dir_id; + char virtual_path[PATH_MAX]; + int ret; + + ret = snprintf(virtual_path, sizeof(virtual_path), "/spotify:/%s", artist); + if ((ret < 0) || (ret >= sizeof(virtual_path))) + { + DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s)\n", artist); + return -1; + } + dir_id = db_directory_addorupdate(virtual_path, 0, DIR_SPOTIFY); + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + return -1; + } + ret = snprintf(virtual_path, sizeof(virtual_path), "/spotify:/%s/%s", artist, album); + if ((ret < 0) || (ret >= sizeof(virtual_path))) + { + DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s/%s)\n", artist, album); + return -1; + } + dir_id = db_directory_addorupdate(virtual_path, 0, dir_id); + if (dir_id <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + return -1; + } + + return dir_id; +} + static int spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_added) { @@ -631,7 +673,6 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde char url[1024]; int ret; int dir_id; - char virtual_path[PATH_MAX]; if (!fptr_sp_track_is_loaded(track)) @@ -681,31 +722,10 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde return -1; } - ret = snprintf(virtual_path, sizeof(virtual_path), "/spotify:/%s", mfi.artist); - if ((ret < 0) || (ret >= sizeof(virtual_path))) - { - DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s)\n", mfi.artist); - free_mfi(&mfi, 1); - return -1; - } - dir_id = db_directory_addorupdate(virtual_path, 0, DIR_SPOTIFY); + dir_id = prepare_directories(mfi.artist, mfi.album); if (dir_id <= 0) { - DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); - free_mfi(&mfi, 1); - return -1; - } - ret = snprintf(virtual_path, sizeof(virtual_path), "/spotify:/%s/%s", mfi.artist, mfi.album); - if ((ret < 0) || (ret >= sizeof(virtual_path))) - { - DPRINTF(E_LOG, L_SPOTIFY, "Virtual path exceeds PATH_MAX (/spotify:/%s/%s)\n", mfi.artist, mfi.album); - free_mfi(&mfi, 1); - return -1; - } - dir_id = db_directory_addorupdate(virtual_path, 0, dir_id); - if (dir_id <= 0) - { - DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory '%s'\n", virtual_path); + DPRINTF(E_LOG, L_SPOTIFY, "Could not add or update directory for item: '%s'\n", url); free_mfi(&mfi, 1); return -1; } From 4bd42dda41a464734620c6334f41d771fa0a222f Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 1 Jan 2017 08:32:05 +0100 Subject: [PATCH 04/33] [spotify] Remove loading saved tracks from the webapi (will be readded later) --- src/spotify.c | 514 +------------------------------------------------- 1 file changed, 10 insertions(+), 504 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index ccb94461..fedc0945 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -47,6 +47,7 @@ #include #include "spotify.h" +#include "spotify_webapi.h" #include "logger.h" #include "misc.h" #include "http.h" @@ -132,19 +133,6 @@ struct artwork_get_param int is_loaded; }; -struct pending_metadata -{ - sp_link *link; - sp_track *track; - struct pending_metadata *next; -}; - -struct reload_list -{ - char *uri; - struct reload_list *next; -}; - /* --- Globals --- */ // Spotify thread static pthread_t tid_spotify; @@ -170,10 +158,6 @@ static enum spotify_state g_state; static int spotify_base_plid; // The base playlist id for Spotify saved tracks in the db static int spotify_saved_plid; -// Linked list of tracks where we are waiting for metadata -static struct pending_metadata *spotify_pending_metadata; -// Linked list of saved tracks which we want to reload at startup -static struct reload_list *spotify_reload_list; // Audio fifo static audio_fifo_t *g_audio_fifo; @@ -206,15 +190,6 @@ const uint8_t g_appkey[] = { 0x09, }; -// Endpoints and credentials for the web api -static char *spotify_access_token; -static char *spotify_refresh_token; -static const char *spotify_client_id = "0e684a5422384114a8ae7ac020f01789"; -static const char *spotify_client_secret = "232af95f39014c9ba218285a5c11a239"; -static const char *spotify_auth_uri = "https://accounts.spotify.com/authorize"; -static const char *spotify_token_uri = "https://accounts.spotify.com/api/token"; -static const char *spotify_tracks_uri = "https://api.spotify.com/v1/me/tracks?limit=50"; - // This section defines and assigns function pointers to the libspotify functions // The arguments and return values must be in sync with the spotify api // Please scroll through the ugliness which follows @@ -937,11 +912,8 @@ spotify_playlist_save(sp_playlist *pl) static enum command_state spotify_uri_register(void *arg, int *retval) { - struct playlist_info pli; - struct pending_metadata *pm; sp_link *link; sp_track *track; - int ret; char *uri = arg; @@ -952,32 +924,6 @@ spotify_uri_register(void *arg, int *retval) return COMMAND_END; } - // Must have playlist for these items, otherwise spotify_cleanup_files will delete them again - if (!spotify_saved_plid) - { - memset(&pli, 0, sizeof(struct playlist_info)); - pli.title = "Spotify Saved"; - pli.type = PL_PLAIN; - pli.path = "spotify:savedtracks"; - pli.virtual_path = "/spotify:/Spotify Saved"; - - ret = db_pl_add(&pli, &spotify_saved_plid); - if (ret < 0) - { - DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist for saved tracks\n"); - *retval = -1; - return COMMAND_END; - } - } - - ret = db_pl_add_item_bypath(spotify_saved_plid, uri); - if (ret < 0) - { - DPRINTF(E_LOG, L_SPOTIFY, "Could not add '%s' to spotify:savedtracks\n", uri); - *retval = -1; - return COMMAND_END; - } - link = fptr_sp_link_create_from_string(uri); if (!link) { @@ -994,358 +940,10 @@ spotify_uri_register(void *arg, int *retval) return COMMAND_END; } - // Maybe we already had the track - if (fptr_sp_track_is_loaded(track)) - { - db_file_ping_bymatch(uri, 0); - fptr_sp_link_release(link); - *retval = 0; - return COMMAND_END; - } - - pm = malloc(sizeof(struct pending_metadata)); - if (!pm) - { - DPRINTF(E_LOG, L_SPOTIFY, "Out of memory\n"); - *retval = -1; - return COMMAND_END; - } - - pm->link = link; - pm->track = track; - pm->next = spotify_pending_metadata; - spotify_pending_metadata = pm; - *retval = 0; return COMMAND_END; } -// TODO Maybe use the commands bh instead? -static enum command_state -spotify_pending_process(void *arg, int *retval) -{ - struct pending_metadata *pm; - int i; - - *retval = 0; - if (!spotify_pending_metadata) - return COMMAND_END; - - // Too early - i = 0; - for (pm = spotify_pending_metadata; pm; pm = pm->next) - { - i++; - - if (!fptr_sp_track_is_loaded(pm->track)) - return COMMAND_END; - } - - DPRINTF(E_DBG, L_SPOTIFY, "All %d tracks loaded, now saving\n", i); - - while ((pm = spotify_pending_metadata)) - { - spotify_track_save(0, pm->track, NULL, time(NULL)); - - // Not sure if we should release link here? We are done with it, but maybe - // libspotify will unload the track if we release, and we don't want that - //fptr_sp_link_release(pm->link); - - spotify_pending_metadata = pm->next; - free(pm); - } - - return COMMAND_END; -} - -static enum command_state -spotify_saved_pl_clear_items(void *arg, int *retval) -{ - if (spotify_saved_plid) - db_pl_clear_items(spotify_saved_plid); - - - *retval = 0; - - return COMMAND_END; -} - -static enum command_state -spotify_cleanup_wrapper(void *arg, int *retval) -{ - *retval = spotify_cleanup_files(); - - return COMMAND_END; -} - -/*--------------------- HELPERS FOR SPOTIFY WEB API -------------------------*/ -/* All the below is in the httpd thread */ - -static char * -jparse_str_from_obj(json_object *haystack, const char *key) -{ - json_object *needle; - - if (json_object_object_get_ex(haystack, key, &needle) && json_object_get_type(needle) == json_type_string) - return strdup(json_object_get_string(needle)); - else - return NULL; -} - -static char * -jparse_str_from_str(const char *s, const char *key) -{ - json_object *haystack; - char *val; - - haystack = json_tokener_parse(s); - if (!haystack) - { - DPRINTF(E_LOG, L_SPOTIFY, "JSON parser returned an error\n"); - return NULL; - } - - val = jparse_str_from_obj(haystack, key); - -#ifdef HAVE_JSON_C_OLD - json_object_put(haystack); -#else - if (json_object_put(haystack) != 1) - DPRINTF(E_LOG, L_SPOTIFY, "Memleak: JSON parser did not free object\n"); -#endif - - return val; -} - -// Will find all track Spotify uri's and register them with libspotify. -// Returns the number of tracks found in the json input. "total" will be the -// total reported by Spotify in the response, and "next" will be an allocated -// string with the url of the next page, as reported by Spotify -static int -jparse_and_register_tracks(int *total, char **next, const char *s) -{ - json_object *haystack; - json_object *needle; - json_object *items; - json_object *item; - json_object *track; - char *uri; - int ret; - int len; - int i; - - haystack = json_tokener_parse(s); - if (!haystack) - { - DPRINTF(E_LOG, L_SPOTIFY, "JSON parser returned an error\n"); - return -1; - } - - if (json_object_object_get_ex(haystack, "total", &needle) && json_object_get_type(needle) == json_type_int) - *total = json_object_get_int(needle); - else - *total = -1; - - *next = jparse_str_from_obj(haystack, "next"); - - if (! (json_object_object_get_ex(haystack, "items", &items) && json_object_get_type(items) == json_type_array) ) - { - DPRINTF(E_LOG, L_SPOTIFY, "No items in reply from Spotify. See:\n%s\n", s); - ret = -1; - goto out_free_json; - } - - len = json_object_array_length(items); - - DPRINTF(E_DBG, L_SPOTIFY, "Got %d saved tracks\n", len); - - for (i = 0; i < len; i++) - { - item = json_object_array_get_idx(items, i); - if (! (item && json_object_object_get_ex(item, "track", &track) - && (uri = jparse_str_from_obj(track, "uri")) )) - { - DPRINTF(E_LOG, L_SPOTIFY, "Unexpected JSON: Item %d did not have 'track'->'uri'\n", i); - len--; - continue; - } - - commands_exec_sync(cmdbase, spotify_uri_register, NULL, uri); - - free(uri); - } - - ret = len; - - out_free_json: -#ifdef HAVE_JSON_C_OLD - json_object_put(haystack); -#else - if (json_object_put(haystack) != 1) - DPRINTF(E_LOG, L_SPOTIFY, "Memleak: JSON parser did not free object\n"); -#endif - - return ret; -} - -static int -tokens_get(const char *code, const char *redirect_uri, const char **err) -{ - struct http_client_ctx ctx; - struct keyval kv; - char *param; - char *body; - int ret; - - memset(&kv, 0, sizeof(struct keyval)); - ret = ( (keyval_add(&kv, "grant_type", "authorization_code") == 0) && - (keyval_add(&kv, "code", code) == 0) && - (keyval_add(&kv, "client_id", spotify_client_id) == 0) && - (keyval_add(&kv, "client_secret", spotify_client_secret) == 0) && - (keyval_add(&kv, "redirect_uri", redirect_uri) == 0) ); - if (!ret) - { - *err = "Add parameters to keyval failed"; - ret = -1; - goto out_clear_kv; - } - - param = http_form_urlencode(&kv); - if (!param) - { - *err = "http_form_uriencode() failed"; - ret = -1; - goto out_clear_kv; - } - - memset(&ctx, 0, sizeof(struct http_client_ctx)); - ctx.url = (char *)spotify_token_uri; - ctx.output_body = param; - ctx.input_body = evbuffer_new(); - - ret = http_client_request(&ctx); - if (ret < 0) - { - *err = "Did not get a reply from Spotify"; - goto out_free_input_body; - } - - // 0-terminate for safety - evbuffer_add(ctx.input_body, "", 1); - - body = (char *)evbuffer_pullup(ctx.input_body, -1); - if (!body || (strlen(body) == 0)) - { - *err = "The reply from Spotify is empty or invalid"; - ret = -1; - goto out_free_input_body; - } - - spotify_access_token = jparse_str_from_str(body, "access_token"); - spotify_refresh_token = jparse_str_from_str(body, "refresh_token"); - - if (!spotify_access_token || !spotify_refresh_token) - { - DPRINTF(E_LOG, L_SPOTIFY, "Could not find token in reply: %s\n", body); - - *err = "Could not find token in Spotify reply (see log)"; - ret = -1; - goto out_free_input_body; - } - - ret = 0; - - out_free_input_body: - evbuffer_free(ctx.input_body); - free(param); - out_clear_kv: - keyval_clear(&kv); - - return ret; -} - -static int -saved_tracks_get(int *total, const char **err, const char *uri) -{ - struct http_client_ctx ctx; - struct keyval kv; - char bearer_token[1024]; - char *body; - char *next; - int ret; - int i; - - *total = -1; - - snprintf(bearer_token, sizeof(bearer_token), "Bearer %s", spotify_access_token); - - memset(&kv, 0, sizeof(struct keyval)); - if (keyval_add(&kv, "Authorization", bearer_token) < 0) - { - *err = "Add bearer_token to keyval failed"; - return -1; - } - - memset(&ctx, 0, sizeof(struct http_client_ctx)); - ctx.output_headers = &kv; - ctx.input_body = evbuffer_new(); - ctx.url = uri; - - next = NULL; - for (i = 0; i < SPOTIFY_WEB_REQUESTS_MAX; i++) - { - ret = http_client_request(&ctx); - if (ret < 0) - { - *err = "Request for saved tracks/albums failed"; - break; - } - - // 0-terminate for safety - evbuffer_add(ctx.input_body, "", 1); - - body = (char *)evbuffer_pullup(ctx.input_body, -1); - if (!body || (strlen(body) == 0)) - { - *err = "Request for saved tracks/albums failed, response was empty"; - ret = -1; - break; - } - - if (next) - free(next); - next = NULL; - - if (uri == spotify_tracks_uri) - ret = jparse_and_register_tracks(total, &next, body); - else - ret = -1; - - if (ret < 0) - { - *err = "Could not parse track/album response from Spotify"; - break; - } - - ret += 50 * i; // Equals total number of tracks/albums registered - - if (!next || (strncmp(next, "null", 4) == 0)) - break; - - ctx.url = next; - - evbuffer_drain(ctx.input_body, evbuffer_get_length(ctx.input_body)); - } - - if (next) - free(next); - - evbuffer_free(ctx.input_body); - keyval_clear(&kv); - - return ret; -} - /* -------------------------- PLAYLIST CALLBACKS ------------------------- */ /** @@ -2105,8 +1703,6 @@ notify_main_thread(sp_session *sess) static void metadata_updated(sp_session *session) { DPRINTF(E_DBG, L_SPOTIFY, "Session metadata updated\n"); - - commands_exec_async(cmdbase, spotify_pending_process, NULL); } /* Misc connection error callbacks */ @@ -2119,20 +1715,9 @@ static void play_token_lost(sp_session *sess) static void connectionstate_updated(sp_session *session) { - struct reload_list *reload; - int ret; - if (SP_CONNECTION_STATE_LOGGED_IN == fptr_sp_session_connectionstate(session)) { DPRINTF(E_LOG, L_SPOTIFY, "Connection to Spotify (re)established, reloading saved tracks\n"); - - while ((reload = spotify_reload_list)) - { - spotify_uri_register(reload->uri, &ret); - spotify_reload_list = reload->next; - free(reload->uri); - free(reload); - } } else if (g_state == SPOTIFY_STATE_PLAYING) { @@ -2186,42 +1771,6 @@ static sp_session_config spconfig = { /* ------------------------------- MAIN LOOP ------------------------------- */ /* Thread: spotify */ -static struct reload_list * -reload_list_create(int plid) -{ - struct query_params qp; - struct db_media_file_info dbmfi; - struct reload_list *head; - struct reload_list *reload; - int ret; - - memset(&qp, 0, sizeof(struct query_params)); - - qp.type = Q_PLITEMS; - qp.sort = S_NONE; - qp.id = plid; - - ret = db_query_start(&qp); - if (ret < 0) - { - db_query_end(&qp); - return NULL; - } - - head = NULL; - while (((ret = db_query_fetch_file(&qp, &dbmfi)) == 0) && (dbmfi.path)) - { - reload = malloc(sizeof(struct reload_list)); - reload->uri = strdup(dbmfi.path); - reload->next = head; - head = reload; - } - - db_query_end(&qp); - - return head; -} - static void * spotify(void *arg) { @@ -2416,35 +1965,18 @@ spotify_artwork_get(struct evbuffer *evbuf, char *path, int max_w, int max_h) void spotify_oauth_interface(struct evbuffer *evbuf, const char *redirect_uri) { - struct keyval kv; - char *param; - int ret; + char *uri; - memset(&kv, 0, sizeof(struct keyval)); - ret = ( (keyval_add(&kv, "client_id", spotify_client_id) == 0) && - (keyval_add(&kv, "response_type", "code") == 0) && - (keyval_add(&kv, "redirect_uri", redirect_uri) == 0) && - (keyval_add(&kv, "scope", "playlist-read-private user-library-read") == 0) && - (keyval_add(&kv, "show_dialog", "false") == 0) ); - if (!ret) - { - DPRINTF(E_LOG, L_SPOTIFY, "Cannot display Spotify oath interface (error adding parameters to keyval)\n"); - goto out_clear_kv; - } - - param = http_form_urlencode(&kv); - if (!param) + uri = spotifywebapi_oauth_uri_get(redirect_uri); + if (!uri) { DPRINTF(E_LOG, L_SPOTIFY, "Cannot display Spotify oath interface (http_form_uriencode() failed)\n"); - goto out_clear_kv; + return; } - evbuffer_add_printf(evbuf, "Click here to authorize forked-daapd with Spotify\n", spotify_auth_uri, param); + evbuffer_add_printf(evbuf, "Click here to authorize forked-daapd with Spotify\n", uri); - free(param); - - out_clear_kv: - keyval_clear(&kv); + free(uri); } /* Thread: httpd */ @@ -2453,7 +1985,6 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch { const char *code; const char *err; - int total; int ret; code = evhttp_find_header(param, "code"); @@ -2467,42 +1998,24 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch evbuffer_add_printf(evbuf, "

Requesting access token from Spotify...\n"); - ret = tokens_get(code, redirect_uri, &err); + ret = spotifywebapi_token_get(code, redirect_uri, &err); if (ret < 0) { evbuffer_add_printf(evbuf, "failed

\n

Error: %s

\n", err); return; } - commands_exec_sync(cmdbase, spotify_saved_pl_clear_items, NULL, NULL); - - evbuffer_add_printf(evbuf, "ok

\n

Retrieving saved tracks...\n"); - - ret = saved_tracks_get(&total, &err, spotify_tracks_uri); - if (ret < 0) - { - evbuffer_add_printf(evbuf, "failed

\n

Error: %s

\n", err); - return; - } - - evbuffer_add_printf(evbuf, "ok, got %d out of %d tracks

\n", ret, total); - - evbuffer_add_printf(evbuf, "

Purging removed tracks/albums...\n"); - - // TODO release links to the items we are going to clean up - - commands_exec_sync(cmdbase, spotify_cleanup_wrapper, NULL, NULL); + // TODO Trigger init-scan after successful access to spotifywebapi evbuffer_add_printf(evbuf, "ok, all done

\n"); return; } -/* Thread: filescanner */ +/* Thread: library */ void spotify_login(char *path) { - struct playlist_info *pli; sp_error err; char *username; char *password; @@ -2555,13 +2068,6 @@ spotify_login(char *path) } else { - pli = db_pl_fetch_bypath("spotify:savedtracks"); - if (pli) - { - spotify_reload_list = reload_list_create(pli->id); - free_pli(pli, 0); - } - db_spotify_purge(); spotify_saved_plid = 0; From 263edaa8b397b99b9f4ca7ec8cbd676b6f85fce3 Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 1 Jan 2017 10:23:34 +0100 Subject: [PATCH 05/33] [misc] Helper function safe_strdup (NULL safe strdup) --- src/misc.c | 9 +++++++++ src/misc.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/misc.c b/src/misc.c index b2856051..0abd07f1 100644 --- a/src/misc.c +++ b/src/misc.c @@ -261,6 +261,15 @@ safe_hextou64(const char *str, uint64_t *val) return 0; } +char * +safe_strdup(const char *str) +{ + if (str == NULL) + return NULL; + + return strdup(str); +} + /* Key/value functions */ struct keyval * diff --git a/src/misc.h b/src/misc.h index 7aa615c1..148fd4b2 100644 --- a/src/misc.h +++ b/src/misc.h @@ -41,6 +41,8 @@ safe_atou64(const char *str, uint64_t *val); int safe_hextou64(const char *str, uint64_t *val); +char * +safe_strdup(const char *str); /* Key/value functions */ struct keyval * From 1672b6704053641f673d7e4031da1c1b80cb14fd Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 1 Jan 2017 11:05:18 +0100 Subject: [PATCH 06/33] [library/filescanner] Move purging of old content to library; Remove ping for spotify files in filescanner (not needed, either they get updated through the init-scan or purged) --- src/filescanner.c | 16 ---------------- src/library.c | 32 +++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 2b526c50..267c1126 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -827,23 +827,7 @@ bulk_scan(int flags) } else { - /* Protect spotify from the imminent purge if rescanning */ - db_transaction_begin(); - db_file_ping_bymatch("spotify:", 0); - db_pl_ping_bymatch("spotify:", 0); - db_transaction_end(); - - DPRINTF(E_DBG, L_SCAN, "Purging old database content\n"); - db_purge_cruft(start); - db_groups_cleanup(); - db_queue_cleanup(); - - cache_artwork_purge_cruft(start); - DPRINTF(E_LOG, L_SCAN, "Bulk library scan completed in %.f sec\n", difftime(end, start)); - - DPRINTF(E_DBG, L_SCAN, "Running post library scan jobs\n"); - db_hook_post_scan(); } } diff --git a/src/library.c b/src/library.c index b3ab3592..bb9b060f 100644 --- a/src/library.c +++ b/src/library.c @@ -37,6 +37,7 @@ #include #include +#include "cache.h" #include "commands.h" #include "conffile.h" #include "db.h" @@ -87,7 +88,7 @@ sort_tag_create(char **sort_tag, char *src_tag) if (*sort_tag) { - DPRINTF(E_DBG, L_SCAN, "Existing sort tag will be normalized: %s\n", *sort_tag); + DPRINTF(E_DBG, L_LIB, "Existing sort tag will be normalized: %s\n", *sort_tag); o_ptr = u8_normalize(UNINORM_NFD, (uint8_t *)*sort_tag, strlen(*sort_tag) + 1, NULL, &len); free(*sort_tag); *sort_tag = (char *)o_ptr; @@ -362,7 +363,7 @@ library_process_media(const char *path, time_t mtime, off_t size, enum data_kind mfi = (struct media_file_info*)malloc(sizeof(struct media_file_info)); if (!mfi) { - DPRINTF(E_LOG, L_SCAN, "Out of memory for mfi\n"); + DPRINTF(E_LOG, L_LIB, "Out of memory for mfi\n"); return; } @@ -377,14 +378,14 @@ library_process_media(const char *path, time_t mtime, off_t size, enum data_kind mfi->fname = strdup(filename); if (!mfi->fname) { - DPRINTF(E_LOG, L_SCAN, "Out of memory for fname\n"); + DPRINTF(E_LOG, L_LIB, "Out of memory for fname\n"); goto out; } mfi->path = strdup(path); if (!mfi->path) { - DPRINTF(E_LOG, L_SCAN, "Out of memory for path\n"); + DPRINTF(E_LOG, L_LIB, "Out of memory for path\n"); goto out; } @@ -407,7 +408,7 @@ library_process_media(const char *path, time_t mtime, off_t size, enum data_kind ret = scan_metadata_ffmpeg(path, mfi); if (ret < 0) { - DPRINTF(E_LOG, L_SCAN, "Playlist URL '%s' is unavailable for probe/metadata, assuming MP3 encoding\n", path); + DPRINTF(E_LOG, L_LIB, "Playlist URL '%s' is unavailable for probe/metadata, assuming MP3 encoding\n", path); mfi->type = strdup("mp3"); mfi->codectype = strdup("mpeg"); mfi->description = strdup("MPEG audio file"); @@ -429,13 +430,13 @@ library_process_media(const char *path, time_t mtime, off_t size, enum data_kind } else { - DPRINTF(E_LOG, L_SCAN, "Unknown scan type for '%s', this error should not occur\n", path); + DPRINTF(E_LOG, L_LIB, "Unknown scan type for '%s', this error should not occur\n", path); ret = -1; } if (ret < 0) { - DPRINTF(E_INFO, L_SCAN, "Could not extract metadata for '%s'\n", path); + DPRINTF(E_INFO, L_LIB, "Could not extract metadata for '%s'\n", path); goto out; } @@ -544,6 +545,18 @@ library_add_playlist_info(const char *path, const char *title, const char *virtu return plid; } +static void +purge_cruft(time_t start) +{ + DPRINTF(E_DBG, L_LIB, "Purging old library content\n"); + db_purge_cruft(start); + db_groups_cleanup(); + db_queue_cleanup(); + + DPRINTF(E_DBG, L_LIB, "Purging old artwork content\n"); + cache_artwork_purge_cruft(start); +} + static enum command_state rescan(void *arg, int *ret) { @@ -633,6 +646,11 @@ initscan() sources[i]->initscan(); } + purge_cruft(starttime); + + DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); + db_hook_post_scan(); + endtime = time(NULL); DPRINTF(E_LOG, L_LIB, "Library init scan completed in %.f sec\n", difftime(endtime, starttime)); From 0bea83cafa46189b8c94d1933b98b8b0b65bf0a3 Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 1 Jan 2017 11:06:00 +0100 Subject: [PATCH 07/33] [spotify] Scan saved albums and playlist using the spotify web api --- src/cache.c | 2 +- src/cache.h | 2 +- src/spotify.c | 262 +++++++++++++++++++++++++++++++++++++------ src/spotify_webapi.c | 16 ++- src/spotify_webapi.h | 8 +- 5 files changed, 247 insertions(+), 43 deletions(-) diff --git a/src/cache.c b/src/cache.c index a42ce018..66bdd0ce 100644 --- a/src/cache.c +++ b/src/cache.c @@ -1415,7 +1415,7 @@ cache_daap_threshold(void) * @return 0 if successful, -1 if an error occurred */ void -cache_artwork_ping(char *path, time_t mtime, int del) +cache_artwork_ping(const char *path, time_t mtime, int del) { struct cache_arg *cmdarg; diff --git a/src/cache.h b/src/cache.h index 16b42350..7e4e112f 100644 --- a/src/cache.h +++ b/src/cache.h @@ -31,7 +31,7 @@ cache_daap_threshold(void); #define CACHE_ARTWORK_INDIVIDUAL 1 void -cache_artwork_ping(char *path, time_t mtime, int del); +cache_artwork_ping(const char *path, time_t mtime, int del); int cache_artwork_delete_by_path(char *path); diff --git a/src/spotify.c b/src/spotify.c index fedc0945..9eb93127 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -156,8 +156,8 @@ static void *g_libhandle; static enum spotify_state g_state; // The base playlist id for all Spotify playlists in the db static int spotify_base_plid; -// The base playlist id for Spotify saved tracks in the db -static int spotify_saved_plid; +// Flag telling us if access to the web api was granted +static bool spotify_access_token_valid; // Audio fifo static audio_fifo_t *g_audio_fifo; @@ -707,7 +707,7 @@ spotify_track_save(int plid, sp_track *track, const char *pltitle, int time_adde // DPRINTF(E_DBG, L_SPOTIFY, "Saving track '%s': '%s' by %s (%s)\n", url, mfi.title, mfi.artist, mfi.album); - library_process_media(url, time(NULL), 0, DATA_KIND_SPOTIFY, 0, 0, &mfi, dir_id); + library_process_media(url, time(NULL), 0, DATA_KIND_SPOTIFY, 0, false, &mfi, dir_id); free_mfi(&mfi, 1); @@ -910,7 +910,7 @@ spotify_playlist_save(sp_playlist *pl) // told which track it was...). Note that this function will result in a ref // count on the sp_link, which the caller must decrease with sp_link_release. static enum command_state -spotify_uri_register(void *arg, int *retval) +uri_register(void *arg, int *retval) { sp_link *link; sp_track *track; @@ -963,7 +963,8 @@ static void playlist_update_in_progress(sp_playlist *pl, bool done, void *userda { DPRINTF(E_DBG, L_SPOTIFY, "Playlist update (status %d): %s\n", done, fptr_sp_playlist_name(pl)); - spotify_playlist_save(pl); + if (!spotify_access_token_valid) // TODO Trigger update of library on pl change + spotify_playlist_save(pl); } } @@ -971,7 +972,8 @@ static void playlist_metadata_updated(sp_playlist *pl, void *userdata) { DPRINTF(E_DBG, L_SPOTIFY, "Playlist metadata updated: %s\n", fptr_sp_playlist_name(pl)); - spotify_playlist_save(pl); + if (!spotify_access_token_valid) // TODO Trigger update of library on pl change + spotify_playlist_save(pl); } /** @@ -1001,7 +1003,8 @@ static void playlist_added(sp_playlistcontainer *pc, sp_playlist *pl, fptr_sp_playlist_add_callbacks(pl, &pl_callbacks, NULL); - spotify_playlist_save(pl); + if (!spotify_access_token_valid) // TODO Trigger update of library on pl change + spotify_playlist_save(pl); } /** @@ -1027,6 +1030,9 @@ playlist_removed(sp_playlistcontainer *pc, sp_playlist *pl, int position, void * fptr_sp_playlist_remove_callbacks(pl, &pl_callbacks, NULL); + if (spotify_access_token_valid) // TODO Trigger update of library on pl change + return; + link = fptr_sp_link_create_from_playlist(pl); if (!link) { @@ -1545,11 +1551,8 @@ artwork_get(void *arg, int *retval) static void logged_in(sp_session *sess, sp_error error) { - cfg_t *spotify_cfg; sp_playlist *pl; sp_playlistcontainer *pc; - struct playlist_info pli; - int ret; int i; if (SP_ERROR_OK != error) @@ -1565,24 +1568,6 @@ logged_in(sp_session *sess, sp_error error) pl = fptr_sp_session_starred_create(sess); fptr_sp_playlist_add_callbacks(pl, &pl_callbacks, NULL); - spotify_cfg = cfg_getsec(cfg, "spotify"); - if (! cfg_getbool(spotify_cfg, "base_playlist_disable")) - { - memset(&pli, 0, sizeof(struct playlist_info)); - pli.title = "Spotify"; - pli.type = PL_FOLDER; - pli.path = "spotify:playlistfolder"; - - ret = db_pl_add(&pli, &spotify_base_plid); - if (ret < 0) - { - DPRINTF(E_LOG, L_SPOTIFY, "Error adding base playlist\n"); - return; - } - } - else - spotify_base_plid = 0; - pc = fptr_sp_session_playlistcontainer(sess); fptr_sp_playlistcontainer_add_callbacks(pc, &pc_callbacks, NULL); @@ -2005,6 +1990,9 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch return; } + // Received a valid access token + spotify_access_token_valid = true; + // TODO Trigger init-scan after successful access to spotifywebapi evbuffer_add_printf(evbuf, "ok, all done

\n"); @@ -2012,6 +2000,15 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch return; } +static void +spotify_uri_register(const char *uri) +{ + char *tmp; + + tmp = strdup(uri); + commands_exec_async(cmdbase, uri_register, tmp); +} + /* Thread: library */ void spotify_login(char *path) @@ -2055,9 +2052,6 @@ spotify_login(char *path) if (path) { - db_spotify_purge(); - spotify_saved_plid = 0; - ret = spotify_file_read(path, &username, &password); if (ret < 0) return; @@ -2068,9 +2062,6 @@ spotify_login(char *path) } else { - db_spotify_purge(); - spotify_saved_plid = 0; - err = fptr_sp_session_relogin(g_sess); } @@ -2081,11 +2072,212 @@ spotify_login(char *path) } } +static void +map_track_to_mfi(const struct spotify_track *track, struct media_file_info* mfi) +{ + mfi->title = safe_strdup(track->name); + mfi->album = safe_strdup(track->album); + mfi->artist = safe_strdup(track->artist); + mfi->album_artist = safe_strdup(track->album_artist); + mfi->disc = track->disc_number; + mfi->song_length = track->duration_ms; + mfi->track = track->track_number; + mfi->compilation = track->is_compilation; +} + +static void +map_album_to_mfi(const struct spotify_album *album, struct media_file_info* mfi) +{ + mfi->album = safe_strdup(album->name); + mfi->album_artist = safe_strdup(album->artist); + mfi->genre = safe_strdup(album->genre); + mfi->compilation = album->is_compilation; +} + +/* Thread: library */ +static int +scan_saved_albums() +{ + struct spotify_request request; + json_object *jsontracks; + int track_count; + struct spotify_album album; + struct spotify_track track; + struct media_file_info mfi; + int dir_id; + int i; + int ret; + + db_transaction_begin(); + + memset(&request, 0, sizeof(struct spotify_request)); + + while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_ALBUMS)) + { + while (0 == spotifywebapi_saved_albums_fetch(&request, &jsontracks, &track_count, &album)) + { + DPRINTF(E_DBG, L_SPOTIFY, "Got saved album: '%s' - '%s' (%s) - track-count: %d\n", + album.artist, album.name, album.uri, track_count); + + dir_id = prepare_directories(album.artist, album.name); + ret = 0; + for (i = 0; i < track_count && ret == 0; i++) + { + ret = spotifywebapi_album_track_fetch(jsontracks, i, &track); + if (ret == 0 && track.uri) + { + memset(&mfi, 0, sizeof(struct media_file_info)); + map_track_to_mfi(&track, &mfi); + map_album_to_mfi(&album, &mfi); + + library_process_media(track.uri, album.mtime, 0, DATA_KIND_SPOTIFY, 0, album.is_compilation, &mfi, dir_id); + spotify_uri_register(track.uri); + + cache_artwork_ping(track.uri, album.mtime, 0); + + free_mfi(&mfi, 1); + } + } + } + } + + spotifywebapi_request_end(&request); + + db_transaction_end(); + + return 0; +} + +/* Thread: library */ +static int +scan_playlisttracks(struct spotify_playlist *playlist, int plid) +{ + struct spotify_request request; + struct spotify_track track; + struct media_file_info mfi; + int dir_id; + + memset(&request, 0, sizeof(struct spotify_request)); + + while (0 == spotifywebapi_request_next(&request, playlist->tracks_href)) + { +// DPRINTF(E_DBG, L_SPOTIFY, "Playlist tracks\n%s\n", request.response_body); + while (0 == spotifywebapi_playlisttracks_fetch(&request, &track)) + { + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist track: '%s' (%s) \n", track.name, track.uri); + + if (track.uri) + { + dir_id = prepare_directories(track.album_artist, track.album); + + memset(&mfi, 0, sizeof(struct media_file_info)); + map_track_to_mfi(&track, &mfi); + + library_process_media(track.uri, 1 /* TODO passing one prevents overwriting existing entries */, 0, DATA_KIND_SPOTIFY, 0, track.is_compilation, &mfi, dir_id); + spotify_uri_register(track.uri); + + cache_artwork_ping(track.uri, 1, 0); + + free_mfi(&mfi, 1); + + db_pl_add_item_bypath(plid, track.uri); + } + } + } + + spotifywebapi_request_end(&request); + + return 0; +} + +/* Thread: library */ +static int +scan_playlists() +{ + struct spotify_request request; + struct spotify_playlist playlist; + char virtual_path[PATH_MAX]; + int plid; + + db_transaction_begin(); + + memset(&request, 0, sizeof(struct spotify_request)); + + while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_PLAYLISTS)) + { + while (0 == spotifywebapi_playlists_fetch(&request, &playlist)) + { + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + if (playlist.owner) + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); + } + else + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); + } + + plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + + if (plid) + scan_playlisttracks(&playlist, plid); + } + } + + spotifywebapi_request_end(&request); + db_transaction_end(); + + return 0; +} + /* Thread: library */ static int spotify_initscan() { + cfg_t *spotify_cfg; + int ret; + + /* Refresh access token for the spotify webapi */ + spotify_access_token_valid = (0 == spotifywebapi_token_refresh()); + if (!spotify_access_token_valid) + { + DPRINTF(E_LOG, L_SPOTIFY, "Spotify webapi token refresh failed. " + "In order to use the web api, authorize forked-daapd to access " + "your saved tracks by visiting http://forked-daapd.local:3689/oauth\n"); + + db_spotify_purge(); + } + + /* + * Add playlist folder for all spotify playlists + */ + spotify_base_plid = 0; + spotify_cfg = cfg_getsec(cfg, "spotify"); + if (! cfg_getbool(spotify_cfg, "base_playlist_disable")) + { + ret = library_add_playlist_info("spotify:playlistfolder", "Spotify", NULL, PL_FOLDER, 0, 0); + if (ret < 0) + DPRINTF(E_LOG, L_SPOTIFY, "Error adding base playlist\n"); + else + spotify_base_plid = ret; + } + + /* + * Login to spotify needs to be done before scanning tracks from the web api. + * (Scanned tracks need to be registered with libspotify for playback) + */ spotify_login(NULL); + + /* + * Scan saved tracks from the web api + */ + if (spotify_access_token_valid) + { + scan_saved_albums(); + scan_playlists(); + } + return 0; } @@ -2112,6 +2304,8 @@ spotify_init(void) sp_error err; int ret; + spotify_access_token_valid = false; + /* Initialize libspotify */ g_libhandle = dlopen("libspotify.so", RTLD_LAZY); if (!g_libhandle) diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index 7184ffcc..e97d1d78 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -392,14 +392,14 @@ spotifywebapi_request_uri(struct spotify_request *request, const char *uri) snprintf(bearer_token, sizeof(bearer_token), "Bearer %s", spotify_access_token); if (keyval_add(request->ctx->output_headers, "Authorization", bearer_token) < 0) { - DPRINTF(E_LOG, L_SPOTIFY, "Add bearer_token to keyval failed"); + DPRINTF(E_LOG, L_SPOTIFY, "Add bearer_token to keyval failed\n"); return -1; } ret = http_client_request(request->ctx); if (ret < 0) { - DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed"); + DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed\n"); return -1; } @@ -409,10 +409,12 @@ spotifywebapi_request_uri(struct spotify_request *request, const char *uri) request->response_body = (char *) evbuffer_pullup(request->ctx->input_body, -1); if (!request->response_body || (strlen(request->response_body) == 0)) { - DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed, response was empty"); + DPRINTF(E_LOG, L_SPOTIFY, "Request for saved tracks/albums failed, response was empty\n"); return -1; } + //DPRINTF(E_DBG, L_SPOTIFY, "Wep api response for '%s'\n%s\n", uri, request->response_body); + request->haystack = json_tokener_parse(request->response_body); if (!request->haystack) { @@ -498,6 +500,8 @@ track_metadata(json_object* jsontrack, struct spotify_track* track) track->artist = jparse_artist_from_obj(jsonartists); } track->disc_number = jparse_int_from_obj(jsontrack, "disc_number"); + track->album_type = jparse_str_from_obj(jsonalbum, "album_type"); + track->is_compilation = (track->album_type && 0 == strcmp(track->album_type, "compilation")); track->duration_ms = jparse_int_from_obj(jsontrack, "duration_ms"); track->name = jparse_str_from_obj(jsontrack, "name"); track->track_number = jparse_int_from_obj(jsontrack, "track_number"); @@ -586,9 +590,13 @@ album_metadata(json_object *jsonalbum, struct spotify_album *album) album->id = jparse_str_from_obj(jsonalbum, "id"); album->album_type = jparse_str_from_obj(jsonalbum, "album_type"); - album->genre = jparse_str_from_obj(jsonalbum, "genre"); // FIXME Genre is an array of strings ('genres'), but it seems to be always empty + album->is_compilation = (album->album_type && 0 == strcmp(album->album_type, "compilation")); + album->label = jparse_str_from_obj(jsonalbum, "label"); album->release_date = jparse_str_from_obj(jsonalbum, "release_date"); + + // TODO Genre is an array of strings ('genres'), but it is always empty (https://github.com/spotify/web-api/issues/157) + //album->genre = jparse_str_from_obj(jsonalbum, "genre"); } int diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h index 455d0758..81cd00ad 100644 --- a/src/spotify_webapi.h +++ b/src/spotify_webapi.h @@ -22,15 +22,14 @@ #include - -#include "http.h" - #ifdef HAVE_JSON_C_OLD # include #else # include #endif +#include +#include "http.h" #define SPOTIFY_WEBAPI_SAVED_TRACKS "https://api.spotify.com/v1/me/tracks?limit=50" #define SPOTIFY_WEBAPI_TRACKS "https://api.spotify.com/v1/tracks/" @@ -43,6 +42,7 @@ struct spotify_album time_t mtime; const char *album_type; + bool is_compilation; const char *artist; const char *genre; const char *id; @@ -61,6 +61,8 @@ struct spotify_track const char *album_artist; const char *artist; int disc_number; + const char *album_type; + bool is_compilation; int duration_ms; const char *id; const char *name; From ae1a45bacc44a821150c9f0d6b100586650d6464 Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 1 Jan 2017 16:51:21 +0100 Subject: [PATCH 08/33] [library/spotify] Implement rescan in spotify.c and purge old files after rescan --- src/db.c | 6 +++--- src/db.h | 2 +- src/filescanner.c | 8 +++++++- src/library.c | 11 ++++++++--- src/spotify.c | 29 +++++++++++++++++++++++------ 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/db.c b/src/db.c index 9bcf0453..d0f79b75 100644 --- a/src/db.c +++ b/src/db.c @@ -3528,12 +3528,12 @@ db_directory_addorupdate(char *virtual_path, int disabled, int parent_id) } void -db_directory_ping_bymatch(char *path) +db_directory_ping_bymatch(char *virtual_path) { -#define Q_TMPL_DIR "UPDATE directories SET db_timestamp = %" PRIi64 " WHERE virtual_path = '/file:%q' OR virtual_path LIKE '/file:%q/%%';" +#define Q_TMPL_DIR "UPDATE directories SET db_timestamp = %" PRIi64 " WHERE virtual_path = '%q' OR virtual_path LIKE '%q/%%';" char *query; - query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), path, path); + query = sqlite3_mprintf(Q_TMPL_DIR, (int64_t)time(NULL), virtual_path, virtual_path); db_query_run(query, 1, 0); #undef Q_TMPL_DIR diff --git a/src/db.h b/src/db.h index 6e2c21b3..079e1044 100644 --- a/src/db.h +++ b/src/db.h @@ -633,7 +633,7 @@ int db_directory_addorupdate(char *virtual_path, int disabled, int parent_id); void -db_directory_ping_bymatch(char *path); +db_directory_ping_bymatch(char *virtual_path); void db_directory_disable_bymatch(char *path, char *strip, uint32_t cookie); diff --git a/src/filescanner.c b/src/filescanner.c index 267c1126..34cd9169 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -766,6 +766,8 @@ bulk_scan(int flags) time_t end; int parent_id; int i; + char virtual_path[PATH_MAX]; + int ret; start = time(NULL); @@ -793,7 +795,11 @@ bulk_scan(int flags) db_file_ping_bymatch(path, 1); db_pl_ping_bymatch(path, 1); - db_directory_ping_bymatch(path); + ret = snprintf(virtual_path, sizeof(virtual_path), "/file:%s", path); + if ((ret < 0) || (ret >= sizeof(virtual_path))) + DPRINTF(E_LOG, L_SCAN, "Virtual path exceeds PATH_MAX (/file:%s)\n", path); + else + db_directory_ping_bymatch(virtual_path); continue; } diff --git a/src/library.c b/src/library.c index bb9b060f..12a56c7a 100644 --- a/src/library.c +++ b/src/library.c @@ -572,6 +572,8 @@ rescan(void *arg, int *ret) sources[i]->rescan(); } + purge_cruft(starttime); + endtime = time(NULL); DPRINTF(E_LOG, L_LIB, "Library rescan completed in %.f sec\n", difftime(endtime, starttime)); @@ -646,10 +648,13 @@ initscan() sources[i]->initscan(); } - purge_cruft(starttime); + if (! (cfg_getbool(cfg_getsec(cfg, "library"), "filescan_disable"))) + { + purge_cruft(starttime); - DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); - db_hook_post_scan(); + DPRINTF(E_DBG, L_LIB, "Running post library scan jobs\n"); + db_hook_post_scan(); + } endtime = time(NULL); DPRINTF(E_LOG, L_LIB, "Library init scan completed in %.f sec\n", difftime(endtime, starttime)); diff --git a/src/spotify.c b/src/spotify.c index 9eb93127..d0b06b3d 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2233,7 +2233,7 @@ scan_playlists() /* Thread: library */ static int -spotify_initscan() +initscan() { cfg_t *spotify_cfg; int ret; @@ -2283,14 +2283,31 @@ spotify_initscan() /* Thread: library */ static int -spotify_rescan() +rescan() { + /* + * Scan saved tracks from the web api + */ + if (spotify_access_token_valid) + { + scan_saved_albums(); + scan_playlists(); + } + else + { + db_transaction_begin(); + db_file_ping_bymatch("spotify:", 0); + db_pl_ping_bymatch("spotify:", 0); + db_directory_ping_bymatch("/spotify:"); + db_transaction_end(); + } + return 0; } /* Thread: library */ static int -spotify_fullrescan() +fullrescan() { return 0; } @@ -2498,7 +2515,7 @@ struct library_source spotifyscanner = .disabled = 0, .init = spotify_init, .deinit = spotify_deinit, - .rescan = spotify_rescan, - .initscan = spotify_initscan, - .fullrescan = spotify_fullrescan, + .rescan = rescan, + .initscan = initscan, + .fullrescan = fullrescan, }; From adac1d3b5fbcc185a15efa05cd412a156d610bdc Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 4 Jan 2017 20:27:55 +0100 Subject: [PATCH 09/33] [spotify] Trigger scan after retrieving a valid access token from the webinterface Library now offers the function "library_exec_async". This allows library sources to offer functions that will be executed in the library thread. Useful to allow partial scans of only one source. --- src/library.c | 15 +++++++++++++++ src/library.h | 4 ++++ src/spotify.c | 14 +++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/library.c b/src/library.c index 12a56c7a..e60776eb 100644 --- a/src/library.c +++ b/src/library.c @@ -689,6 +689,21 @@ library_is_exiting() return scan_exit; } +/* + * Execute the function 'func' with the given argument 'arg' in the library thread. + * + * The pointer passed as argument is freed in the library thread after func returned. + * + * @param func The function to be executed + * @param arg Argument passed to func + * @return 0 if triggering the function execution succeeded, -1 on failure. + */ +int +library_exec_async(command_function func, void *arg) +{ + return commands_exec_async(cmdbase, func, arg); +} + static void * library(void *arg) { diff --git a/src/library.h b/src/library.h index b703fbdf..b07a64bf 100644 --- a/src/library.h +++ b/src/library.h @@ -23,6 +23,7 @@ #include #include +#include "commands.h" #include "db.h" /* @@ -88,6 +89,9 @@ library_set_scanning(bool is_scanning); bool library_is_exiting(); +int +library_exec_async(command_function func, void *arg); + int library_init(); diff --git a/src/spotify.c b/src/spotify.c index d0b06b3d..005e04df 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -413,6 +413,9 @@ fptr_assign_all() // End of ugly part +static enum command_state +scan_webapi(void *arg, int *ret); + /* ------------------------------- MISC HELPERS ---------------------------- */ static int @@ -1993,7 +1996,8 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch // Received a valid access token spotify_access_token_valid = true; - // TODO Trigger init-scan after successful access to spotifywebapi + // Trigger scan after successful access to spotifywebapi + library_exec_async(scan_webapi, NULL); evbuffer_add_printf(evbuf, "ok, all done

\n"); @@ -2312,6 +2316,14 @@ fullrescan() return 0; } +/* Thread: httpd */ +static enum command_state +scan_webapi(void *arg, int *ret) +{ + *ret = rescan(); + return COMMAND_END; +} + /* Thread: main */ int spotify_init(void) From 99945fa5763c9322bd409a6ee944ed59a3b18ef5 Mon Sep 17 00:00:00 2001 From: chme Date: Thu, 5 Jan 2017 22:09:19 +0100 Subject: [PATCH 10/33] [spotify] Rescan of single playlists if update trigger received from libspotify (Readds "spotify:savedtracks" playlist to avoid deletion of saved tracks) --- src/spotify.c | 215 +++++++++++++++++++++++++++++++++++++------ src/spotify_webapi.c | 93 ++++++++++++++++++- src/spotify_webapi.h | 3 +- 3 files changed, 280 insertions(+), 31 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index 005e04df..0781b534 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -158,6 +158,11 @@ static enum spotify_state g_state; static int spotify_base_plid; // Flag telling us if access to the web api was granted static bool spotify_access_token_valid; +// The base playlist id for Spotify saved tracks in the db +static int spotify_saved_plid; + +// Flag to avoid triggering playlist change events while the (re)scan is running +static bool scanning; // Audio fifo static audio_fifo_t *g_audio_fifo; @@ -414,7 +419,11 @@ fptr_assign_all() static enum command_state -scan_webapi(void *arg, int *ret); +webapi_scan(void *arg, int *ret); +static enum command_state +webapi_pl_save(void *arg, int *ret); +static enum command_state +webapi_pl_remove(void *arg, int *ret); /* ------------------------------- MISC HELPERS ---------------------------- */ @@ -947,6 +956,33 @@ uri_register(void *arg, int *retval) return COMMAND_END; } +static void +webapi_playlist_updated(sp_playlist *pl) +{ + sp_link *link; + char url[1024]; + int ret; + + if (!scanning) + { + // Run playlist save in the library thread + link = fptr_sp_link_create_from_playlist(pl); + if (!link) + { + DPRINTF(E_LOG, L_SPOTIFY, "Could not create link for playlist: '%s'\n", fptr_sp_playlist_name(pl)); + return; + } + + ret = fptr_sp_link_as_string(link, url, sizeof(url)); + if (ret == sizeof(url)) + { + DPRINTF(E_DBG, L_SPOTIFY, "Spotify link truncated: %s\n", url); + } + fptr_sp_link_release(link); + + library_exec_async(webapi_pl_save, strdup(url)); + } +} /* -------------------------- PLAYLIST CALLBACKS ------------------------- */ /** @@ -966,8 +1002,14 @@ static void playlist_update_in_progress(sp_playlist *pl, bool done, void *userda { DPRINTF(E_DBG, L_SPOTIFY, "Playlist update (status %d): %s\n", done, fptr_sp_playlist_name(pl)); - if (!spotify_access_token_valid) // TODO Trigger update of library on pl change - spotify_playlist_save(pl); + if (spotify_access_token_valid) + { + webapi_playlist_updated(pl); + } + else + { + spotify_playlist_save(pl); + } } } @@ -975,8 +1017,15 @@ static void playlist_metadata_updated(sp_playlist *pl, void *userdata) { DPRINTF(E_DBG, L_SPOTIFY, "Playlist metadata updated: %s\n", fptr_sp_playlist_name(pl)); - if (!spotify_access_token_valid) // TODO Trigger update of library on pl change - spotify_playlist_save(pl); + if (spotify_access_token_valid) + { + //TODO Update disabled to prevent multiple triggering of updates e. g. on adding a playlist + //webapi_playlist_updated(pl); + } + else + { + spotify_playlist_save(pl); + } } /** @@ -1006,8 +1055,37 @@ static void playlist_added(sp_playlistcontainer *pc, sp_playlist *pl, fptr_sp_playlist_add_callbacks(pl, &pl_callbacks, NULL); - if (!spotify_access_token_valid) // TODO Trigger update of library on pl change - spotify_playlist_save(pl); + if (spotify_access_token_valid) + { + webapi_playlist_updated(pl); + } + else + { + spotify_playlist_save(pl); + } +} + +static int +playlist_remove(const char *uri) +{ + struct playlist_info *pli; + int plid; + + pli = db_pl_fetch_bypath(uri); + + if (!pli) + { + DPRINTF(E_DBG, L_SPOTIFY, "Playlist %s not found, can't delete\n", uri); + return -1; + } + + plid = pli->id; + + free_pli(pli, 0); + + db_spotify_pl_delete(plid); + spotify_cleanup_files(); + return 0; } /** @@ -1023,19 +1101,14 @@ static void playlist_added(sp_playlistcontainer *pc, sp_playlist *pl, static void playlist_removed(sp_playlistcontainer *pc, sp_playlist *pl, int position, void *userdata) { - struct playlist_info *pli; sp_link *link; char url[1024]; - int plid; int ret; DPRINTF(E_INFO, L_SPOTIFY, "Playlist removed: %s\n", fptr_sp_playlist_name(pl)); fptr_sp_playlist_remove_callbacks(pl, &pl_callbacks, NULL); - if (spotify_access_token_valid) // TODO Trigger update of library on pl change - return; - link = fptr_sp_link_create_from_playlist(pl); if (!link) { @@ -1050,21 +1123,16 @@ playlist_removed(sp_playlistcontainer *pc, sp_playlist *pl, int position, void * } fptr_sp_link_release(link); - pli = db_pl_fetch_bypath(url); - - if (!pli) + if (spotify_access_token_valid) { - DPRINTF(E_DBG, L_SPOTIFY, "Playlist %s not found, can't delete\n", url); - return; + // Run playlist remove in the library thread + if (!scanning) + library_exec_async(webapi_pl_remove, strdup(url)); + } + else + { + playlist_remove(url); } - - plid = pli->id; - - free_pli(pli, 0); - - db_spotify_pl_delete(plid); - - spotify_cleanup_files(); } /** @@ -1997,7 +2065,7 @@ spotify_oauth_callback(struct evbuffer *evbuf, struct evkeyvalq *param, const ch spotify_access_token_valid = true; // Trigger scan after successful access to spotifywebapi - library_exec_async(scan_webapi, NULL); + library_exec_async(webapi_scan, NULL); evbuffer_add_printf(evbuf, "ok, all done

\n"); @@ -2140,6 +2208,9 @@ scan_saved_albums() cache_artwork_ping(track.uri, album.mtime, 0); free_mfi(&mfi, 1); + + if (spotify_saved_plid) + db_pl_add_item_bypath(spotify_saved_plid, track.uri); } } } @@ -2224,8 +2295,10 @@ scan_playlists() plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); - if (plid) + if (plid > 0) scan_playlisttracks(&playlist, plid); + else + DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); } } @@ -2235,6 +2308,58 @@ scan_playlists() return 0; } +/* Thread: library */ +static int +scan_playlist(const char *uri) +{ + struct spotify_request request; + struct spotify_playlist playlist; + char virtual_path[PATH_MAX]; + int plid; + + db_transaction_begin(); + + memset(&request, 0, sizeof(struct spotify_request)); + + if (0 == spotifywebapi_playlist_start(&request, uri, &playlist)) + { + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + if (playlist.owner) + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); + } + else + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); + } + + plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + + if (plid > 0) + scan_playlisttracks(&playlist, plid); + else + DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); + } + + spotifywebapi_request_end(&request); + db_transaction_end(); + + return 0; +} + +static void +create_saved_tracks_playlist() +{ + spotify_saved_plid = library_add_playlist_info("spotify:savedtracks", "Spotify Saved", "/spotify:/Spotify Saved", PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + + if (spotify_saved_plid <= 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist for saved tracks\n"); + spotify_saved_plid = 0; + } +} + /* Thread: library */ static int initscan() @@ -2242,6 +2367,8 @@ initscan() cfg_t *spotify_cfg; int ret; + scanning = true; + /* Refresh access token for the spotify webapi */ spotify_access_token_valid = (0 == spotifywebapi_token_refresh()); if (!spotify_access_token_valid) @@ -2257,6 +2384,7 @@ initscan() * Add playlist folder for all spotify playlists */ spotify_base_plid = 0; + spotify_saved_plid = 0; spotify_cfg = cfg_getsec(cfg, "spotify"); if (! cfg_getbool(spotify_cfg, "base_playlist_disable")) { @@ -2278,10 +2406,13 @@ initscan() */ if (spotify_access_token_valid) { + create_saved_tracks_playlist(); scan_saved_albums(); scan_playlists(); } + scanning = false; + return 0; } @@ -2289,11 +2420,14 @@ initscan() static int rescan() { + scanning = true; + /* * Scan saved tracks from the web api */ if (spotify_access_token_valid) { + create_saved_tracks_playlist(); scan_saved_albums(); scan_playlists(); } @@ -2306,6 +2440,8 @@ rescan() db_transaction_end(); } + scanning = false; + return 0; } @@ -2316,14 +2452,34 @@ fullrescan() return 0; } -/* Thread: httpd */ +/* Thread: library */ static enum command_state -scan_webapi(void *arg, int *ret) +webapi_scan(void *arg, int *ret) { *ret = rescan(); return COMMAND_END; } +/* Thread: library */ +static enum command_state +webapi_pl_save(void *arg, int *ret) +{ + const char *uri = arg; + + *ret = scan_playlist(uri); + return COMMAND_END; +} + +/* Thread: library */ +static enum command_state +webapi_pl_remove(void *arg, int *ret) +{ + const char *uri = arg; + + *ret = playlist_remove(uri); + return COMMAND_END; +} + /* Thread: main */ int spotify_init(void) @@ -2334,6 +2490,7 @@ spotify_init(void) int ret; spotify_access_token_valid = false; + scanning = false; /* Initialize libspotify */ g_libhandle = dlopen("libspotify.so", RTLD_LAZY); diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index e97d1d78..889df1d6 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -50,6 +50,7 @@ static const char *spotify_client_id = "0e684a5422384114a8ae7ac020f01789"; static const char *spotify_client_secret = "232af95f39014c9ba218285a5c11a239"; static const char *spotify_auth_uri = "https://accounts.spotify.com/authorize"; static const char *spotify_token_uri = "https://accounts.spotify.com/api/token"; +static const char *spotify_playlist_uri = "https://api.spotify.com/v1/users/%s/playlists/%s"; /*--------------------- HELPERS FOR SPOTIFY WEB API -------------------------*/ @@ -690,18 +691,67 @@ spotifywebapi_playlists_fetch(struct spotify_request *request, struct spotify_pl if (request->index >= request->count) { + DPRINTF(E_DBG, L_SPOTIFY, "All playlists processed\n"); return -1; } jsonplaylist = json_object_array_get_idx(request->items, request->index); + if (!jsonplaylist) + { + DPRINTF(E_LOG, L_SPOTIFY, "Error fetching playlist at index '%d'\n", request->index); + return -1; + } playlist_metadata(jsonplaylist, playlist); - request->index++; return 0; } +/* + * Extracts the owner and the id from a spotify playlist uri + * + * Playlist-uri has the following format: spotify:user:[owner]:playlist:[id] + * Owner and plid must be freed by the caller. + */ +static int +get_owner_plid_from_uri(const char *uri, char **owner, char **plid) +{ + char *ptr1; + char *ptr2; + char *tmp; + size_t len; + + ptr1 = strchr(uri, ':'); + if (!ptr1) + return -1; + ptr1++; + ptr1 = strchr(ptr1, ':'); + if (!ptr1) + return -1; + ptr1++; + ptr2 = strchr(ptr1, ':'); + + len = ptr2 - ptr1; + + tmp = malloc(sizeof(char) * (len + 1)); + strncpy(tmp, ptr1, len); + tmp[len] = '\0'; + *owner = tmp; + + ptr2++; + ptr1 = strchr(ptr2, ':'); + if (!ptr1) + { + free(tmp); + return -1; + } + ptr1++; + *plid = strdup(ptr1); + + return 0; +} + int spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spotify_track *track) { @@ -731,3 +781,44 @@ spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spoti return 0; } + +int +spotifywebapi_playlist_start(struct spotify_request *request, const char *path, struct spotify_playlist *playlist) +{ + char uri[1024]; + char *owner; + char *id; + int ret; + + ret = get_owner_plid_from_uri(path, &owner, &id); + if (ret < 0) + { + DPRINTF(E_LOG, L_SPOTIFY, "Error extracting owner and id from playlist uri '%s'\n", path); + return -1; + } + + ret = snprintf(uri, sizeof(uri), spotify_playlist_uri, owner, id); + if (ret < 0 || ret >= sizeof(uri)) + { + DPRINTF(E_LOG, L_SPOTIFY, "Error creating playlist endpoint uri for playlist '%s'\n", path); + free(owner); + free(id); + return -1; + } + + ret = spotifywebapi_request_uri(request, uri); + if (ret < 0) + { + free(owner); + free(id); + return -1; + } + + request->haystack = json_tokener_parse(request->response_body); + playlist_metadata(request->haystack, playlist); + + free(owner); + free(id); + return 0; +} + diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h index 81cd00ad..bd96ba22 100644 --- a/src/spotify_webapi.h +++ b/src/spotify_webapi.h @@ -121,6 +121,7 @@ int spotifywebapi_playlists_fetch(struct spotify_request *request, struct spotify_playlist* playlist); int spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spotify_track *track); - +int +spotifywebapi_playlist_start(struct spotify_request *request, const char *path, struct spotify_playlist *playlist); #endif /* SRC_SPOTIFY_WEBAPI_H_ */ From 65ce902e3c80e4fedf2211c3c0963290b3af8c80 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 10:15:58 +0100 Subject: [PATCH 11/33] [db_init] Set admint.value to VARCHAR(255) Only a cosmetic change to avoid future confusion. sqlite3 ignores the number of chars given to VARCHAR (see https://sqlite.org/faq.html#q9 ), a db upgrade is therefor not necessary. --- src/db_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db_init.c b/src/db_init.c index b7f473a9..d3017841 100644 --- a/src/db_init.c +++ b/src/db_init.c @@ -28,7 +28,7 @@ #define T_ADMIN \ "CREATE TABLE IF NOT EXISTS admin(" \ " key VARCHAR(32) PRIMARY KEY NOT NULL," \ - " value VARCHAR(32) NOT NULL" \ + " value VARCHAR(255) NOT NULL" \ ");" #define T_FILES \ From 4e60626749c2d57957981f0cb22dc937185be236 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 10:45:25 +0100 Subject: [PATCH 12/33] [db] Add missing purge of directories in db_purge_all --- src/db.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index d0f79b75..7d8e8eae 100644 --- a/src/db.c +++ b/src/db.c @@ -781,7 +781,8 @@ db_purge_cruft(time_t ref) void db_purge_all(void) { -#define Q_TMPL "DELETE FROM playlists WHERE type <> %d;" +#define Q_TMPL_PL "DELETE FROM playlists WHERE type <> %d;" +#define Q_TMPL_DIR "DELETE FROM directories WHERE id >= %d;" char *queries[4] = { "DELETE FROM inotify;", @@ -809,7 +810,8 @@ db_purge_all(void) DPRINTF(E_DBG, L_DB, "Purged %d rows\n", sqlite3_changes(hdl)); } - query = sqlite3_mprintf(Q_TMPL, PL_SPECIAL); + // Purge playlists + query = sqlite3_mprintf(Q_TMPL_PL, PL_SPECIAL); if (!query) { DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); @@ -829,7 +831,31 @@ db_purge_all(void) DPRINTF(E_DBG, L_DB, "Purged %d rows\n", sqlite3_changes(hdl)); sqlite3_free(query); -#undef Q_TMPL + + // Purge directories + query = sqlite3_mprintf(Q_TMPL_DIR, DIR_MAX); + if (!query) + { + DPRINTF(E_LOG, L_DB, "Out of memory for query string\n"); + return; + } + + DPRINTF(E_DBG, L_DB, "Running purge query '%s'\n", query); + + ret = db_exec(query, &errmsg); + if (ret != SQLITE_OK) + { + DPRINTF(E_LOG, L_DB, "Purge query '%s' error: %s\n", query, errmsg); + + sqlite3_free(errmsg); + } + else + DPRINTF(E_DBG, L_DB, "Purged %d rows\n", sqlite3_changes(hdl)); + + sqlite3_free(query); + +#undef Q_TMPL_PL +#undef Q_TMPL_DIR } static int From 510d38c059086bde35bc1a2afa68603ee4d8c12f Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 11:08:47 +0100 Subject: [PATCH 13/33] [library/filescanner/spotify] Rework full-rescan logic --- src/filescanner.c | 6 +--- src/library.c | 34 +++++++++++++++++++-- src/spotify.c | 77 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 34cd9169..8e7be450 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -1482,13 +1482,9 @@ filescanner_fullrescan() { DPRINTF(E_LOG, L_SCAN, "Full rescan triggered\n"); - player_playback_stop(); - db_queue_clear(); inofd_event_unset(); // Clears all inotify watches - db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups - - inofd_event_set(); bulk_scan(F_SCAN_BULK); + inofd_event_set(); return 0; } diff --git a/src/library.c b/src/library.c index e60776eb..1cbb6176 100644 --- a/src/library.c +++ b/src/library.c @@ -43,6 +43,7 @@ #include "db.h" #include "logger.h" #include "misc.h" +#include "player.h" static struct commands_base *cmdbase; @@ -564,12 +565,21 @@ rescan(void *arg, int *ret) time_t endtime; int i; + DPRINTF(E_LOG, L_LIB, "Library rescan triggered\n"); + starttime = time(NULL); for (i = 0; sources[i]; i++) { if (!sources[i]->disabled && sources[i]->rescan) - sources[i]->rescan(); + { + DPRINTF(E_INFO, L_LIB, "Rescan library source '%s'\n", sources[i]->name); + sources[i]->rescan(); + } + else + { + DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", sources[i]->name); + } } purge_cruft(starttime); @@ -585,14 +595,34 @@ rescan(void *arg, int *ret) static enum command_state fullrescan(void *arg, int *ret) { + time_t starttime; + time_t endtime; int i; + DPRINTF(E_LOG, L_LIB, "Library full-rescan triggered\n"); + + starttime = time(NULL); + + player_playback_stop(); + db_queue_clear(); + db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups + for (i = 0; sources[i]; i++) { if (!sources[i]->disabled && sources[i]->fullrescan) - sources[i]->fullrescan(); + { + DPRINTF(E_INFO, L_LIB, "Full-rescan library source '%s'\n", sources[i]->name); + sources[i]->fullrescan(); + } + else + { + DPRINTF(E_INFO, L_LIB, "Library source '%s' is disabled\n", sources[i]->name); + } } + endtime = time(NULL); + DPRINTF(E_LOG, L_LIB, "Library full-rescan completed in %.f sec\n", difftime(endtime, starttime)); + scanning = false; *ret = 0; return COMMAND_END; diff --git a/src/spotify.c b/src/spotify.c index 0781b534..d8cea6e0 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2180,8 +2180,6 @@ scan_saved_albums() int i; int ret; - db_transaction_begin(); - memset(&request, 0, sizeof(struct spotify_request)); while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_ALBUMS)) @@ -2191,6 +2189,8 @@ scan_saved_albums() DPRINTF(E_DBG, L_SPOTIFY, "Got saved album: '%s' - '%s' (%s) - track-count: %d\n", album.artist, album.name, album.uri, track_count); + db_transaction_begin(); + dir_id = prepare_directories(album.artist, album.name); ret = 0; for (i = 0; i < track_count && ret == 0; i++) @@ -2213,13 +2213,13 @@ scan_saved_albums() db_pl_add_item_bypath(spotify_saved_plid, track.uri); } } + + db_transaction_end(); } } spotifywebapi_request_end(&request); - db_transaction_end(); - return 0; } @@ -2274,8 +2274,6 @@ scan_playlists() char virtual_path[PATH_MAX]; int plid; - db_transaction_begin(); - memset(&request, 0, sizeof(struct spotify_request)); while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_PLAYLISTS)) @@ -2284,6 +2282,8 @@ scan_playlists() { DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + db_transaction_begin(); + if (playlist.owner) { snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); @@ -2299,11 +2299,12 @@ scan_playlists() scan_playlisttracks(&playlist, plid); else DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + db_transaction_end(); } } spotifywebapi_request_end(&request); - db_transaction_end(); return 0; } @@ -2317,14 +2318,14 @@ scan_playlist(const char *uri) char virtual_path[PATH_MAX]; int plid; - db_transaction_begin(); - memset(&request, 0, sizeof(struct spotify_request)); if (0 == spotifywebapi_playlist_start(&request, uri, &playlist)) { DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + db_transaction_begin(); + if (playlist.owner) { snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); @@ -2340,10 +2341,11 @@ scan_playlist(const char *uri) scan_playlisttracks(&playlist, plid); else DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + db_transaction_end(); } spotifywebapi_request_end(&request); - db_transaction_end(); return 0; } @@ -2360,13 +2362,28 @@ create_saved_tracks_playlist() } } -/* Thread: library */ -static int -initscan() +static void +create_base_playlist() { cfg_t *spotify_cfg; int ret; + spotify_base_plid = 0; + spotify_cfg = cfg_getsec(cfg, "spotify"); + if (!cfg_getbool(spotify_cfg, "base_playlist_disable")) + { + ret = library_add_playlist_info("spotify:playlistfolder", "Spotify", NULL, PL_FOLDER, 0, 0); + if (ret < 0) + DPRINTF(E_LOG, L_SPOTIFY, "Error adding base playlist\n"); + else + spotify_base_plid = ret; + } +} + +/* Thread: library */ +static int +initscan() +{ scanning = true; /* Refresh access token for the spotify webapi */ @@ -2383,17 +2400,9 @@ initscan() /* * Add playlist folder for all spotify playlists */ - spotify_base_plid = 0; + create_base_playlist(); + spotify_saved_plid = 0; - spotify_cfg = cfg_getsec(cfg, "spotify"); - if (! cfg_getbool(spotify_cfg, "base_playlist_disable")) - { - ret = library_add_playlist_info("spotify:playlistfolder", "Spotify", NULL, PL_FOLDER, 0, 0); - if (ret < 0) - DPRINTF(E_LOG, L_SPOTIFY, "Error adding base playlist\n"); - else - spotify_base_plid = ret; - } /* * Login to spotify needs to be done before scanning tracks from the web api. @@ -2422,6 +2431,8 @@ rescan() { scanning = true; + create_base_playlist(); + /* * Scan saved tracks from the web api */ @@ -2449,6 +2460,26 @@ rescan() static int fullrescan() { + scanning = true; + + create_base_playlist(); + + /* + * Scan saved tracks from the web api + */ + if (spotify_access_token_valid) + { + create_saved_tracks_playlist(); + scan_saved_albums(); + scan_playlists(); + } + else + { + spotify_login(NULL); + } + + scanning = false; + return 0; } From 8ee3ef5b1572924b30a651315f6f7efc30ab27b8 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 11:46:11 +0100 Subject: [PATCH 14/33] [filescanner] Trigger library-scan on .init-rescan / .full-rescan --- src/filescanner.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/filescanner.c b/src/filescanner.c index 8e7be450..41fe876a 100644 --- a/src/filescanner.c +++ b/src/filescanner.c @@ -483,7 +483,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ DPRINTF(E_LOG, L_SCAN, "Startup rescan triggered, found init-rescan file: %s\n", file); - filescanner_rescan(); + library_rescan(); break; case FILE_CTRL_FULLSCAN: @@ -492,7 +492,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ DPRINTF(E_LOG, L_SCAN, "Full rescan triggered, found full-rescan file: %s\n", file); - filescanner_fullrescan(); + library_fullrescan(); break; default: @@ -1470,7 +1470,6 @@ filescanner_rescan() inofd_event_unset(); // Clears all inotify watches db_watch_clear(); - inofd_event_set(); bulk_scan(F_SCAN_BULK | F_SCAN_RESCAN); @@ -1483,8 +1482,8 @@ filescanner_fullrescan() DPRINTF(E_LOG, L_SCAN, "Full rescan triggered\n"); inofd_event_unset(); // Clears all inotify watches - bulk_scan(F_SCAN_BULK); inofd_event_set(); + bulk_scan(F_SCAN_BULK); return 0; } From 66bbcf0576176dd663441c02579df7bb1a9033f9 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 16:33:04 +0100 Subject: [PATCH 15/33] [spotify] Ignore playlist updates with missing uri --- src/spotify.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index d8cea6e0..6dfea9a8 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2322,27 +2322,34 @@ scan_playlist(const char *uri) if (0 == spotifywebapi_playlist_start(&request, uri, &playlist)) { - DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); - - db_transaction_begin(); - - if (playlist.owner) + if (!playlist.uri) { - snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); + DPRINTF(E_LOG, L_SPOTIFY, "Got playlist with missing uri for path:: '%s'\n", uri); } else { - snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + db_transaction_begin(); + + if (playlist.owner) + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); + } + else + { + snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); + } + + plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + + if (plid > 0) + scan_playlisttracks(&playlist, plid); + else + DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); + + db_transaction_end(); } - - plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); - - if (plid > 0) - scan_playlisttracks(&playlist, plid); - else - DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); - - db_transaction_end(); } spotifywebapi_request_end(&request); From ab3582dd694774db10ca5ab826793add483aef3f Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 6 Jan 2017 17:08:30 +0100 Subject: [PATCH 16/33] [spotify_webapi] Cleanup: remove unused functions --- src/spotify_webapi.c | 69 +------------------------------------------- src/spotify_webapi.h | 8 ----- 2 files changed, 1 insertion(+), 76 deletions(-) diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index 889df1d6..b880895e 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -372,7 +372,7 @@ spotifywebapi_token_refresh() return ret; } -int +static int spotifywebapi_request_uri(struct spotify_request *request, const char *uri) { char bearer_token[1024]; @@ -510,73 +510,6 @@ track_metadata(json_object* jsontrack, struct spotify_track* track) track->id = jparse_str_from_obj(jsontrack, "id"); } -int -spotifywebapi_saved_tracks_fetch(struct spotify_request *request, struct spotify_track *track) -{ - json_object *item; - json_object *jsontrack; - - memset(track, 0, sizeof(struct spotify_track)); - - if (request->index >= request->count) - { - return -1; - } - - item = json_object_array_get_idx(request->items, request->index); - if (!(item && json_object_object_get_ex(item, "track", &jsontrack))) - { - DPRINTF(E_LOG, L_SPOTIFY, "Unexpected JSON: Item %d did not have 'track'->'uri'\n", request->index); - request->index++; - return -1; - } - - track_metadata(jsontrack, track); - track->added_at = jparse_str_from_obj(item, "added_at"); -// if (track->added_at) -// strptime("%Y-%m-%dT%H:%M:%SZ"); - - request->index++; - - return 0; -} - -int -spotifywebapi_track(const char *path, struct spotify_track *track) -{ -// char uri[1024]; -// struct http_client_ctx *ctx; -// char *response_body; -// json_object *haystack; -// -// memset(track, 0, sizeof(struct spotify_track)); -// -// if (strlen(path) < 15) // spotify:track:3RZwWEcQHD0Sy1hN1xNYeh -// return -1; -// -// snprintf(uri, sizeof(uri), "%s%s", spotify_track_uri, (path + 14)); -// ctx = request_uri(uri); -// -// if (!ctx) -// return -1; -// -// response_body = (char *) evbuffer_pullup(ctx->input_body, -1); -// if (!response_body || (strlen(response_body) == 0)) -// { -// DPRINTF(E_LOG, L_SPOTIFY, "Request for track failed, response was empty"); -// http_client_ctx_free(ctx); -// return -1; -// } -// -// haystack = json_tokener_parse(response_body); -// track_metadata(haystack, track); -// -// http_client_ctx_free(ctx); -// jparse_free(haystack); - - return 0; -} - void album_metadata(json_object *jsonalbum, struct spotify_album *album) { diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h index bd96ba22..6921efed 100644 --- a/src/spotify_webapi.h +++ b/src/spotify_webapi.h @@ -31,8 +31,6 @@ #include "http.h" -#define SPOTIFY_WEBAPI_SAVED_TRACKS "https://api.spotify.com/v1/me/tracks?limit=50" -#define SPOTIFY_WEBAPI_TRACKS "https://api.spotify.com/v1/tracks/" #define SPOTIFY_WEBAPI_SAVED_ALBUMS "https://api.spotify.com/v1/me/albums?limit=50" #define SPOTIFY_WEBAPI_SAVED_PLAYLISTS "https://api.spotify.com/v1/me/playlists?limit=50" @@ -103,17 +101,11 @@ spotifywebapi_token_get(const char *code, const char *redirect_uri, const char * int spotifywebapi_token_refresh(); -int -spotifywebapi_request_uri(struct spotify_request *request, const char *uri); void spotifywebapi_request_end(struct spotify_request *request); int spotifywebapi_request_next(struct spotify_request *request, const char *uri); int -spotifywebapi_saved_tracks_fetch(struct spotify_request *request, struct spotify_track *track); -int -spotifywebapi_track(const char *path, struct spotify_track *track); -int spotifywebapi_saved_albums_fetch(struct spotify_request *request, json_object **jsontracks, int *track_count, struct spotify_album *album); int spotifywebapi_album_track_fetch(json_object *jsontracks, int index, struct spotify_track *track); From f632789f8b6832323950daab153a8b527ba4b1fe Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 7 Jan 2017 07:22:40 +0100 Subject: [PATCH 17/33] [spotify] Set missing values for year, artwork, type, codectype, description and improve logging --- src/spotify.c | 22 ++++++++++++++++++++-- src/spotify_webapi.c | 21 ++++++++++++++++++++- src/spotify_webapi.h | 2 ++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index 6dfea9a8..b62c863c 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2155,6 +2155,11 @@ map_track_to_mfi(const struct spotify_track *track, struct media_file_info* mfi) mfi->song_length = track->duration_ms; mfi->track = track->track_number; mfi->compilation = track->is_compilation; + + mfi->artwork = ARTWORK_SPOTIFY; + mfi->type = strdup("spotify"); + mfi->codectype = strdup("wav"); + mfi->description = strdup("Spotify audio"); } static void @@ -2164,6 +2169,7 @@ map_album_to_mfi(const struct spotify_album *album, struct media_file_info* mfi) mfi->album_artist = safe_strdup(album->artist); mfi->genre = safe_strdup(album->genre); mfi->compilation = album->is_compilation; + mfi->year = album->release_year; } /* Thread: library */ @@ -2178,8 +2184,10 @@ scan_saved_albums() struct media_file_info mfi; int dir_id; int i; + int count; int ret; + count = 0; memset(&request, 0, sizeof(struct spotify_request)); while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_ALBUMS)) @@ -2215,6 +2223,10 @@ scan_saved_albums() } db_transaction_end(); + + count++; + if (count >= request.total || (count % 10 == 0)) + DPRINTF(E_LOG, L_SPOTIFY, "Scanned %d of %d saved albums\n", count, request.total); } } @@ -2273,14 +2285,16 @@ scan_playlists() struct spotify_playlist playlist; char virtual_path[PATH_MAX]; int plid; + int count; + count = 0; memset(&request, 0, sizeof(struct spotify_request)); while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_PLAYLISTS)) { while (0 == spotifywebapi_playlists_fetch(&request, &playlist)) { - DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); db_transaction_begin(); @@ -2301,6 +2315,10 @@ scan_playlists() DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); db_transaction_end(); + + count++; + if (count >= request.total || (count % 10 == 0)) + DPRINTF(E_LOG, L_SPOTIFY, "Scanned %d of %d saved playlists\n", count, request.total); } } @@ -2328,7 +2346,7 @@ scan_playlist(const char *uri) } else { - DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' (%s) \n", playlist.name, playlist.uri); + DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); db_transaction_begin(); diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index b880895e..5782ba02 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -414,7 +414,7 @@ spotifywebapi_request_uri(struct spotify_request *request, const char *uri) return -1; } - //DPRINTF(E_DBG, L_SPOTIFY, "Wep api response for '%s'\n%s\n", uri, request->response_body); +// DPRINTF(E_DBG, L_SPOTIFY, "Wep api response for '%s'\n%s\n", uri, request->response_body); request->haystack = json_tokener_parse(request->response_body); if (!request->haystack) @@ -510,6 +510,22 @@ track_metadata(json_object* jsontrack, struct spotify_track* track) track->id = jparse_str_from_obj(jsontrack, "id"); } +static int +get_year_from_releasdate(const char *release_date) +{ + char tmp[5]; + uint32_t year = 0; + + if (release_date && strlen(release_date) >= 4) + { + strncpy(tmp, release_date, sizeof(tmp)); + tmp[4] = '\0'; + safe_atou32(tmp, &year); + } + + return year; +} + void album_metadata(json_object *jsonalbum, struct spotify_album *album) { @@ -527,7 +543,10 @@ album_metadata(json_object *jsonalbum, struct spotify_album *album) album->is_compilation = (album->album_type && 0 == strcmp(album->album_type, "compilation")); album->label = jparse_str_from_obj(jsonalbum, "label"); + album->release_date = jparse_str_from_obj(jsonalbum, "release_date"); + album->release_date_precision = jparse_str_from_obj(jsonalbum, "release_date_precision"); + album->release_year = get_year_from_releasdate(album->release_date); // TODO Genre is an array of strings ('genres'), but it is always empty (https://github.com/spotify/web-api/issues/157) //album->genre = jparse_str_from_obj(jsonalbum, "genre"); diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h index 6921efed..af9efaf4 100644 --- a/src/spotify_webapi.h +++ b/src/spotify_webapi.h @@ -47,6 +47,8 @@ struct spotify_album const char *label; const char *name; const char *release_date; + const char *release_date_precision; + int release_year; const char *uri; }; From 03c5ecd69086bea7e99c586eff4659b3bb348259 Mon Sep 17 00:00:00 2001 From: chme Date: Sun, 8 Jan 2017 07:56:41 +0100 Subject: [PATCH 18/33] [spotify] Split into smaller db transactions (do not keep an open transaction between requests) --- src/spotify.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index b62c863c..e8c2237b 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2248,6 +2248,8 @@ scan_playlisttracks(struct spotify_playlist *playlist, int plid) while (0 == spotifywebapi_request_next(&request, playlist->tracks_href)) { + db_transaction_begin(); + // DPRINTF(E_DBG, L_SPOTIFY, "Playlist tracks\n%s\n", request.response_body); while (0 == spotifywebapi_playlisttracks_fetch(&request, &track)) { @@ -2270,6 +2272,8 @@ scan_playlisttracks(struct spotify_playlist *playlist, int plid) db_pl_add_item_bypath(plid, track.uri); } } + + db_transaction_end(); } spotifywebapi_request_end(&request); @@ -2286,8 +2290,10 @@ scan_playlists() char virtual_path[PATH_MAX]; int plid; int count; + int trackcount; count = 0; + trackcount = 0; memset(&request, 0, sizeof(struct spotify_request)); while (0 == spotifywebapi_request_next(&request, SPOTIFY_WEBAPI_SAVED_PLAYLISTS)) @@ -2296,8 +2302,6 @@ scan_playlists() { DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); - db_transaction_begin(); - if (playlist.owner) { snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); @@ -2307,18 +2311,18 @@ scan_playlists() snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); } + db_transaction_begin(); plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + db_transaction_end(); if (plid > 0) scan_playlisttracks(&playlist, plid); else DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); - db_transaction_end(); - count++; - if (count >= request.total || (count % 10 == 0)) - DPRINTF(E_LOG, L_SPOTIFY, "Scanned %d of %d saved playlists\n", count, request.total); + trackcount += playlist.tracks_count; + DPRINTF(E_LOG, L_SPOTIFY, "Scanned %d of %d saved playlists (%d tracks)\n", count, request.total, trackcount); } } @@ -2348,8 +2352,6 @@ scan_playlist(const char *uri) { DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); - db_transaction_begin(); - if (playlist.owner) { snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); @@ -2359,14 +2361,14 @@ scan_playlist(const char *uri) snprintf(virtual_path, PATH_MAX, "/spotify:/%s", playlist.name); } + db_transaction_begin(); plid = library_add_playlist_info(playlist.uri, playlist.name, virtual_path, PL_PLAIN, spotify_base_plid, DIR_SPOTIFY); + db_transaction_end(); if (plid > 0) scan_playlisttracks(&playlist, plid); else DPRINTF(E_LOG, L_SPOTIFY, "Error adding playlist: '%s' (%s) \n", playlist.name, playlist.uri); - - db_transaction_end(); } } From 2cfb4b6a28b1e30c667c6f17a346e0d8db69cc76 Mon Sep 17 00:00:00 2001 From: chme Date: Tue, 10 Jan 2017 22:29:13 +0100 Subject: [PATCH 19/33] [spotify] Simplify parsing of artist/album-artist name forked-daapd does not support a 1:n relationship between tracks and artists. Just take the first artist the spotify web api returns (libspotify does also return only one artist, so this should not introduce a regression). --- src/spotify_webapi.c | 49 +++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index 5782ba02..a3a2c335 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -123,38 +123,21 @@ jparse_time_from_obj(json_object *haystack, const char *key) return parsed_time; } -static char * -jparse_artist_from_obj(json_object *artists) +static const char * +jparse_str_from_array(json_object *array, int index, const char *key) { - char artistnames[1024]; - json_object *artist; + json_object *item; int count; - int i; - const char *tmp; - count = json_object_array_length(artists); - if (count == 0) - { - return NULL; - } + if (json_object_get_type(array) != json_type_array) + return NULL; - for (i = 0; i < count; i++) - { - artist = json_object_array_get_idx(artists, i); - tmp = jparse_str_from_obj(artist, "name"); + count = json_object_array_length(array); + if (count <= 0 || count <= index) + return NULL; - if (i == 0) - { - strcpy(artistnames, tmp); - } - else - { - strncat(artistnames, ", ", sizeof(artistnames) - 2); - strncat(artistnames, tmp, sizeof(artistnames) - strlen(artistnames)); - } - } - - return strdup(artistnames); + item = json_object_array_get_idx(array, index); + return jparse_str_from_obj(item, key); } static void @@ -491,14 +474,14 @@ track_metadata(json_object* jsontrack, struct spotify_track* track) if (json_object_object_get_ex(jsontrack, "album", &jsonalbum)) { track->album = jparse_str_from_obj(jsonalbum, "name"); - if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists)) { - track->album_artist = jparse_artist_from_obj(jsonartists); + track->album_artist = jparse_str_from_array(jsonartists, 0, "name"); } } - if (json_object_object_get_ex(jsontrack, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + if (json_object_object_get_ex(jsontrack, "artists", &jsonartists)) { - track->artist = jparse_artist_from_obj(jsonartists); + track->artist = jparse_str_from_array(jsonartists, 0, "name"); } track->disc_number = jparse_int_from_obj(jsontrack, "disc_number"); track->album_type = jparse_str_from_obj(jsonalbum, "album_type"); @@ -531,9 +514,9 @@ album_metadata(json_object *jsonalbum, struct spotify_album *album) { json_object* jsonartists; - if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists) && json_object_get_type(jsonartists) == json_type_array) + if (json_object_object_get_ex(jsonalbum, "artists", &jsonartists)) { - album->artist = jparse_artist_from_obj(jsonartists); + album->artist = jparse_str_from_array(jsonartists, 0, "name"); } album->name = jparse_str_from_obj(jsonalbum, "name"); album->uri = jparse_str_from_obj(jsonalbum, "uri"); From 9c9c583b8198d3ba4587399d30b4f1babcec3c2c Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 11 Jan 2017 18:14:33 +0100 Subject: [PATCH 20/33] [library] move declaration scan_metadata_ffmpeg back to filescanner.h --- src/filescanner.h | 3 +++ src/library.c | 1 + src/library.h | 3 --- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/filescanner.h b/src/filescanner.h index 2a858f99..1acdf61e 100644 --- a/src/filescanner.h +++ b/src/filescanner.h @@ -14,6 +14,9 @@ /* Actual scanners */ +int +scan_metadata_ffmpeg(const char *file, struct media_file_info *mfi); + int scan_metadata_icy(char *url, struct media_file_info *mfi); diff --git a/src/library.c b/src/library.c index 1cbb6176..9f9c1473 100644 --- a/src/library.c +++ b/src/library.c @@ -41,6 +41,7 @@ #include "commands.h" #include "conffile.h" #include "db.h" +#include "filescanner.h" #include "logger.h" #include "misc.h" #include "player.h" diff --git a/src/library.h b/src/library.h index b07a64bf..e137f520 100644 --- a/src/library.h +++ b/src/library.h @@ -64,9 +64,6 @@ struct library_source }; -int -scan_metadata_ffmpeg(const char *file, struct media_file_info *mfi); - void library_process_media(const char *path, time_t mtime, off_t size, enum data_kind data_kind, enum media_kind force_media_kind, bool force_compilation, struct media_file_info *external_mfi, int dir_id); From 528614909c15d9e45befd25eebcf17fdbb9846cf Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 11 Jan 2017 18:25:49 +0100 Subject: [PATCH 21/33] [filescanner/library] Move filescanner to subfolder --- src/Makefile.am | 8 ++++---- src/library.c | 2 +- src/{ => library}/filescanner.c | 2 +- src/{ => library}/filescanner.h | 0 src/{ => library}/filescanner_ffmpeg.c | 2 +- src/{ => library}/filescanner_itunes.c | 1 - src/{ => library}/filescanner_playlist.c | 2 +- src/{ => library}/filescanner_smartpl.c | 1 - 8 files changed, 8 insertions(+), 10 deletions(-) rename src/{ => library}/filescanner.c (99%) rename src/{ => library}/filescanner.h (100%) rename src/{ => library}/filescanner_ffmpeg.c (99%) rename src/{ => library}/filescanner_itunes.c (99%) rename src/{ => library}/filescanner_playlist.c (99%) rename src/{ => library}/filescanner_smartpl.c (99%) diff --git a/src/Makefile.am b/src/Makefile.am index 08a26531..fb6d93d8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,7 +2,7 @@ sbin_PROGRAMS = forked-daapd if COND_ITUNES -ITUNES_SRC=filescanner_itunes.c +ITUNES_SRC=library/filescanner_itunes.c endif if COND_SPOTIFY @@ -79,9 +79,9 @@ forked_daapd_SOURCES = main.c \ logger.c logger.h \ conffile.c conffile.h \ cache.c cache.h \ - filescanner.c filescanner.h \ - filescanner_ffmpeg.c filescanner_playlist.c \ - filescanner_smartpl.c $(ITUNES_SRC) \ + library/filescanner.c library/filescanner.h \ + library/filescanner_ffmpeg.c library/filescanner_playlist.c \ + library/filescanner_smartpl.c $(ITUNES_SRC) \ library.c library.h \ mdns_avahi.c mdns.h \ remote_pairing.c remote_pairing.h \ diff --git a/src/library.c b/src/library.c index 9f9c1473..2e3509c2 100644 --- a/src/library.c +++ b/src/library.c @@ -41,7 +41,7 @@ #include "commands.h" #include "conffile.h" #include "db.h" -#include "filescanner.h" +#include "library/filescanner.h" #include "logger.h" #include "misc.h" #include "player.h" diff --git a/src/filescanner.c b/src/library/filescanner.c similarity index 99% rename from src/filescanner.c rename to src/library/filescanner.c index 41fe876a..9d2961a1 100644 --- a/src/filescanner.c +++ b/src/library/filescanner.c @@ -54,7 +54,7 @@ #include "logger.h" #include "db.h" -#include "filescanner.h" +#include "library/filescanner.h" #include "conffile.h" #include "misc.h" #include "remote_pairing.h" diff --git a/src/filescanner.h b/src/library/filescanner.h similarity index 100% rename from src/filescanner.h rename to src/library/filescanner.h diff --git a/src/filescanner_ffmpeg.c b/src/library/filescanner_ffmpeg.c similarity index 99% rename from src/filescanner_ffmpeg.c rename to src/library/filescanner_ffmpeg.c index 2528dd55..07cf8dd8 100644 --- a/src/filescanner_ffmpeg.c +++ b/src/library/filescanner_ffmpeg.c @@ -31,8 +31,8 @@ #include #include +#include "db.h" #include "logger.h" -#include "filescanner.h" #include "misc.h" #include "http.h" diff --git a/src/filescanner_itunes.c b/src/library/filescanner_itunes.c similarity index 99% rename from src/filescanner_itunes.c rename to src/library/filescanner_itunes.c index 6abd8420..0896c786 100644 --- a/src/filescanner_itunes.c +++ b/src/library/filescanner_itunes.c @@ -41,7 +41,6 @@ #include "logger.h" #include "db.h" -#include "filescanner.h" #include "conffile.h" #include "misc.h" diff --git a/src/filescanner_playlist.c b/src/library/filescanner_playlist.c similarity index 99% rename from src/filescanner_playlist.c rename to src/library/filescanner_playlist.c index dffcd228..345b05c1 100644 --- a/src/filescanner_playlist.c +++ b/src/library/filescanner_playlist.c @@ -36,7 +36,7 @@ #include "logger.h" #include "db.h" -#include "filescanner.h" +#include "library/filescanner.h" #include "misc.h" #include "library.h" diff --git a/src/filescanner_smartpl.c b/src/library/filescanner_smartpl.c similarity index 99% rename from src/filescanner_smartpl.c rename to src/library/filescanner_smartpl.c index 4eb25585..341a41c6 100644 --- a/src/filescanner_smartpl.c +++ b/src/library/filescanner_smartpl.c @@ -33,7 +33,6 @@ #include "logger.h" #include "db.h" -#include "filescanner.h" #include "misc.h" #include "SMARTPLLexer.h" From a15923c377c8371eb3aa764e168b7a9ac25af176 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 13 Jan 2017 18:28:36 +0100 Subject: [PATCH 22/33] [spotify] Fix for expired spotify access token and some cleanup --- src/spotify.c | 12 ++++++++++-- src/spotify_webapi.c | 45 ++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index e8c2237b..6a1fb6df 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -1075,10 +1075,12 @@ playlist_remove(const char *uri) if (!pli) { - DPRINTF(E_DBG, L_SPOTIFY, "Playlist %s not found, can't delete\n", uri); + DPRINTF(E_LOG, L_SPOTIFY, "Playlist '%s' not found, can't delete\n", uri); return -1; } + DPRINTF(E_LOG, L_SPOTIFY, "Removing playlist '%s' (%s)\n", pli->title, uri); + plid = pli->id; free_pli(pli, 0); @@ -2302,6 +2304,12 @@ scan_playlists() { DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); + if (!playlist.uri || !playlist.name || playlist.tracks_count == 0) + { + DPRINTF(E_DBG, L_SPOTIFY, "Ignoring playlist '%s' with %d tracks (%s)\n", playlist.name, playlist.tracks_count, playlist.uri); + continue; + } + if (playlist.owner) { snprintf(virtual_path, PATH_MAX, "/spotify:/%s (%s)", playlist.name, playlist.owner); @@ -2350,7 +2358,7 @@ scan_playlist(const char *uri) } else { - DPRINTF(E_DBG, L_SPOTIFY, "Got playlist: '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); + DPRINTF(E_LOG, L_SPOTIFY, "Saving playlist '%s' with %d tracks (%s) \n", playlist.name, playlist.tracks_count, playlist.uri); if (playlist.owner) { diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index a3a2c335..8a9b075d 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -20,17 +20,12 @@ #include "spotify_webapi.h" #include +#include #include #include #include #include -#ifdef HAVE_JSON_C_OLD -# include -#else -# include -#endif - #include "db.h" #include "http.h" #include "library.h" @@ -321,7 +316,7 @@ spotifywebapi_token_refresh() const char *err; int ret; - if (token_requested && difftime(token_requested, time(NULL)) < expires_in) + if (token_requested && difftime(time(NULL), token_requested) < expires_in) { DPRINTF(E_DBG, L_SPOTIFY, "Spotify token still valid\n"); return 0; @@ -356,7 +351,7 @@ spotifywebapi_token_refresh() } static int -spotifywebapi_request_uri(struct spotify_request *request, const char *uri) +request_uri(struct spotify_request *request, const char *uri) { char bearer_token[1024]; int ret; @@ -442,7 +437,7 @@ spotifywebapi_request_next(struct spotify_request *request, const char *uri) spotifywebapi_request_end(request); } - ret = spotifywebapi_request_uri(request, next_uri); + ret = request_uri(request, next_uri); free(next_uri); if (ret < 0) @@ -465,8 +460,8 @@ spotifywebapi_request_next(struct spotify_request *request, const char *uri) return 0; } -void -track_metadata(json_object* jsontrack, struct spotify_track* track) +static void +parse_metadata_track(json_object* jsontrack, struct spotify_track* track) { json_object* jsonalbum; json_object* jsonartists; @@ -494,14 +489,14 @@ track_metadata(json_object* jsontrack, struct spotify_track* track) } static int -get_year_from_releasdate(const char *release_date) +get_year_from_date(const char *date) { char tmp[5]; uint32_t year = 0; - if (release_date && strlen(release_date) >= 4) + if (date && strlen(date) >= 4) { - strncpy(tmp, release_date, sizeof(tmp)); + strncpy(tmp, date, sizeof(tmp)); tmp[4] = '\0'; safe_atou32(tmp, &year); } @@ -509,8 +504,8 @@ get_year_from_releasdate(const char *release_date) return year; } -void -album_metadata(json_object *jsonalbum, struct spotify_album *album) +static void +parse_metadata_album(json_object *jsonalbum, struct spotify_album *album) { json_object* jsonartists; @@ -529,7 +524,7 @@ album_metadata(json_object *jsonalbum, struct spotify_album *album) album->release_date = jparse_str_from_obj(jsonalbum, "release_date"); album->release_date_precision = jparse_str_from_obj(jsonalbum, "release_date_precision"); - album->release_year = get_year_from_releasdate(album->release_date); + album->release_year = get_year_from_date(album->release_date); // TODO Genre is an array of strings ('genres'), but it is always empty (https://github.com/spotify/web-api/issues/157) //album->genre = jparse_str_from_obj(jsonalbum, "genre"); @@ -558,7 +553,7 @@ spotifywebapi_saved_albums_fetch(struct spotify_request *request, json_object ** return -1; } - album_metadata(jsonalbum, album); + parse_metadata_album(jsonalbum, album); album->added_at = jparse_str_from_obj(item, "added_at"); album->mtime = jparse_time_from_obj(item, "added_at"); @@ -590,13 +585,13 @@ spotifywebapi_album_track_fetch(json_object *jsontracks, int index, struct spoti return -1; } - track_metadata(jsontrack, track); + parse_metadata_track(jsontrack, track); return 0; } -void -playlist_metadata(json_object *jsonplaylist, struct spotify_playlist *playlist) +static void +parse_metadata_playlist(json_object *jsonplaylist, struct spotify_playlist *playlist) { json_object *needle; @@ -637,7 +632,7 @@ spotifywebapi_playlists_fetch(struct spotify_request *request, struct spotify_pl return -1; } - playlist_metadata(jsonplaylist, playlist); + parse_metadata_playlist(jsonplaylist, playlist); request->index++; return 0; @@ -708,7 +703,7 @@ spotifywebapi_playlisttracks_fetch(struct spotify_request *request, struct spoti return -1; } - track_metadata(jsontrack, track); + parse_metadata_track(jsontrack, track); track->added_at = jparse_str_from_obj(item, "added_at"); track->mtime = jparse_time_from_obj(item, "added_at"); @@ -741,7 +736,7 @@ spotifywebapi_playlist_start(struct spotify_request *request, const char *path, return -1; } - ret = spotifywebapi_request_uri(request, uri); + ret = request_uri(request, uri); if (ret < 0) { free(owner); @@ -750,7 +745,7 @@ spotifywebapi_playlist_start(struct spotify_request *request, const char *path, } request->haystack = json_tokener_parse(request->response_body); - playlist_metadata(request->haystack, playlist); + parse_metadata_playlist(request->haystack, playlist); free(owner); free(id); From 00efed7988daf49c743f55e0a1e9add954365ac2 Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 14 Jan 2017 13:59:14 +0100 Subject: [PATCH 23/33] [filescanner] Fix scanning of fifos --- src/library/filescanner.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/library/filescanner.c b/src/library/filescanner.c index 9d2961a1..f3e5e030 100644 --- a/src/library/filescanner.c +++ b/src/library/filescanner.c @@ -397,6 +397,7 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ bool is_bulkscan; bool is_type_compilation; enum media_kind media_kind; + enum data_kind data_kind; is_bulkscan = (flags & F_SCAN_BULK); @@ -411,7 +412,12 @@ process_file(char *file, time_t mtime, off_t size, int type, int flags, int dir_ else media_kind = 0; - library_process_media(file, mtime, size, DATA_KIND_FILE, media_kind, is_type_compilation, NULL, dir_id); + if (F_SCAN_TYPE_PIPE & type) + data_kind = DATA_KIND_PIPE; + else + data_kind = DATA_KIND_FILE; + + library_process_media(file, mtime, size, data_kind, media_kind, is_type_compilation, NULL, dir_id); cache_artwork_ping(file, mtime, !is_bulkscan); // TODO [artworkcache] If entry in artwork cache exists for no artwork available, delete the entry if media file has embedded artwork From 69c3a9c6dc3bbf2859c0b73014bb182ad6aaa2fa Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 18 Jan 2017 20:10:18 +0100 Subject: [PATCH 24/33] Add missing log domains --- forked-daapd.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forked-daapd.8 b/forked-daapd.8 index e6873b31..04871fd9 100644 --- a/forked-daapd.8 +++ b/forked-daapd.8 @@ -21,7 +21,7 @@ Debug domains; available domains are: \fIconfig\fP, \fIdaap\fP, \fIrsp\fP, \fIscan\fP, \fIxcode\fP, \fIevent\fP, \fIhttp\fP, \fIremote\fP, \fIdacp\fP, \fIffmpeg\fP, \fIartwork\fP, \fIplayer\fP, \fIraop\fP, \fIlaudio\fP, \fIdmap\fP, \fIfdbperf\fP, \fIspotify\fP, \fIlastfm\fP, -\fIcache\fP, \fImpd\fP, \fIstream\fP. +\fIcache\fP, \fImpd\fP, \fIstream\fP, \fIcast\fP, \fIfifo\fP, \fIlib\fP. .TP \fB\-c\fR \fIfile\fP Use \fIfile\fP as the configuration file. From 77086c9b8200e9e1288ef3a007e4c9028f733a7c Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 18 Jan 2017 20:12:14 +0100 Subject: [PATCH 25/33] [library] Check for init/deinit functions in library source --- src/library.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/library.c b/src/library.c index 2e3509c2..a536b7f5 100644 --- a/src/library.c +++ b/src/library.c @@ -31,6 +31,7 @@ #ifdef HAVE_PTHREAD_NP_H # include #endif +#include #include #include #include @@ -795,6 +796,9 @@ library_init(void) for (i = 0; sources[i]; i++) { + if (!sources[i]->init) + continue; + ret = sources[i]->init(); if (ret < 0) sources[i]->disabled = 1; @@ -844,6 +848,7 @@ library_deinit() for (i = 0; sources[i]; i++) { + if (sources[i]->deinit && !sources[i]->disabled) sources[i]->deinit(); } From e8d6b1778437172b93011c9112617896d43f4bed Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 18 Jan 2017 20:28:56 +0100 Subject: [PATCH 26/33] fix indentation --- src/spotify.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spotify.c b/src/spotify.c index 6a1fb6df..fd077aa8 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -1003,11 +1003,11 @@ static void playlist_update_in_progress(sp_playlist *pl, bool done, void *userda DPRINTF(E_DBG, L_SPOTIFY, "Playlist update (status %d): %s\n", done, fptr_sp_playlist_name(pl)); if (spotify_access_token_valid) - { + { webapi_playlist_updated(pl); } else - { + { spotify_playlist_save(pl); } } From a4180a59024cdb2dd0a5f6e2327d71f4f5839b8c Mon Sep 17 00:00:00 2001 From: chme Date: Wed, 18 Jan 2017 20:32:02 +0100 Subject: [PATCH 27/33] [spotify] Increase log level for skipping playlists during scan --- src/spotify.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spotify.c b/src/spotify.c index fd077aa8..70df5a3c 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2306,7 +2306,7 @@ scan_playlists() if (!playlist.uri || !playlist.name || playlist.tracks_count == 0) { - DPRINTF(E_DBG, L_SPOTIFY, "Ignoring playlist '%s' with %d tracks (%s)\n", playlist.name, playlist.tracks_count, playlist.uri); + DPRINTF(E_LOG, L_SPOTIFY, "Ignoring playlist '%s' with %d tracks (%s)\n", playlist.name, playlist.tracks_count, playlist.uri); continue; } From 825236c8bbcb1cfdd38f777ee317d67fa8927f6c Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 20 Jan 2017 16:31:33 +0100 Subject: [PATCH 28/33] [spotify_webapi] Fix memory leaks --- src/spotify_webapi.c | 19 +++++++++++-------- src/spotify_webapi.h | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/spotify_webapi.c b/src/spotify_webapi.c index 8a9b075d..525f0cfb 100644 --- a/src/spotify_webapi.c +++ b/src/spotify_webapi.c @@ -136,7 +136,7 @@ jparse_str_from_array(json_object *array, int index, const char *key) } static void -http_client_ctx_free(struct http_client_ctx *ctx) +free_http_client_ctx(struct http_client_ctx *ctx) { if (!ctx) return; @@ -240,13 +240,19 @@ tokens_get(struct keyval *kv, const char **err) goto out_free_input_body; } + free(spotify_access_token); + spotify_access_token = NULL; + tmp = jparse_str_from_obj(haystack, "access_token"); if (tmp) - spotify_access_token = strdup(tmp); + spotify_access_token = strdup(tmp); tmp = jparse_str_from_obj(haystack, "refresh_token"); if (tmp) - spotify_refresh_token = strdup(tmp); + { + free(spotify_refresh_token); + spotify_refresh_token = strdup(tmp); + } expires_in = jparse_int_from_obj(haystack, "expires_in"); if (expires_in == 0) @@ -408,7 +414,7 @@ request_uri(struct spotify_request *request, const char *uri) void spotifywebapi_request_end(struct spotify_request *request) { - http_client_ctx_free(request->ctx); + free_http_client_ctx(request->ctx); jparse_free(request->haystack); } @@ -416,7 +422,6 @@ int spotifywebapi_request_next(struct spotify_request *request, const char *uri) { char *next_uri; - const char *tmp; int ret; if (request->ctx && !request->next_uri) @@ -444,9 +449,7 @@ spotifywebapi_request_next(struct spotify_request *request, const char *uri) return ret; request->total = jparse_int_from_obj(request->haystack, "total"); - tmp = jparse_str_from_obj(request->haystack, "next"); - if (tmp) - request->next_uri = strdup(tmp); + request->next_uri = jparse_str_from_obj(request->haystack, "next"); if (jparse_array_from_obj(request->haystack, "items", &request->items) < 0) { diff --git a/src/spotify_webapi.h b/src/spotify_webapi.h index af9efaf4..744724bc 100644 --- a/src/spotify_webapi.h +++ b/src/spotify_webapi.h @@ -91,7 +91,7 @@ struct spotify_request json_object *items; int count; int total; - char *next_uri; + const char *next_uri; int index; }; From dea5c50b67cbfd40d35e69c265ce2c9480ace213 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 20 Jan 2017 16:46:26 +0100 Subject: [PATCH 29/33] [player] Fix memleaks found with scan-build --- src/player.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/player.c b/src/player.c index 23bbd789..32225ca2 100644 --- a/src/player.c +++ b/src/player.c @@ -1388,7 +1388,10 @@ source_read(uint8_t *buf, int len, uint64_t rtptime) { ret = source_open(ps, cur_streaming->end + 1, 0); if (ret < 0) - return -1; + { + source_free(ps); + return -1; + } ret = source_play(); if (ret < 0) @@ -2485,6 +2488,7 @@ playback_prev_bh(void *arg, int *retval) ret = source_open(ps, last_rtptime + AIRTUNES_V2_PACKET_SAMPLES, 0); if (ret < 0) { + source_free(ps); playback_abort(); *retval = -1; @@ -2556,6 +2560,7 @@ playback_next_bh(void *arg, int *retval) ret = source_open(ps, last_rtptime + AIRTUNES_V2_PACKET_SAMPLES, 0); if (ret < 0) { + source_free(ps); playback_abort(); *retval = -1; return COMMAND_END; From 4e869f6fda288e53808b59a2edf87acf6e1c19af Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 20 Jan 2017 16:50:09 +0100 Subject: [PATCH 30/33] [dacp] Fix indentation --- src/httpd_dacp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/httpd_dacp.c b/src/httpd_dacp.c index 56a5430f..f290e5f3 100644 --- a/src/httpd_dacp.c +++ b/src/httpd_dacp.c @@ -1074,7 +1074,7 @@ dacp_reply_cue_play(struct evhttp_request *req, struct evbuffer *evbuf, char **u dmap_send_error(req, "cacr", "Playback failed to start"); return; - } + } } else { @@ -1097,9 +1097,9 @@ dacp_reply_cue_play(struct evhttp_request *req, struct evbuffer *evbuf, char **u dmap_send_error(req, "cacr", "Playback failed to start"); return; + } } } - } ret = player_playback_start_byitem(queue_item); free_queue_item(queue_item, 0); From 6163269832efe9df2e822ffcc03a102fad83dc89 Mon Sep 17 00:00:00 2001 From: chme Date: Fri, 20 Jan 2017 17:03:45 +0100 Subject: [PATCH 31/33] [dacp/db] Fix issue found with scan-build - Result of operation is garbage or undefined in dacp_reply_cue_play - Uninitialized argument value in dacp_reply_playqueuecontents - Uninitialized argument value in queue_fetch_byposrelativetoitem --- src/db.c | 5 ++++- src/httpd_dacp.c | 13 ++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/db.c b/src/db.c index 7d8e8eae..ae2c2ca5 100644 --- a/src/db.c +++ b/src/db.c @@ -4691,7 +4691,10 @@ queue_fetch_byposrelativetoitem(int pos, uint32_t item_id, char shuffle, struct ret = queue_fetch_bypos(pos_absolute, shuffle, queue_item, with_metadata); - DPRINTF(E_DBG, L_DB, "Fetch by pos: fetched item (id=%d, pos=%d, file-id=%d)\n", queue_item->id, queue_item->pos, queue_item->file_id); + if (ret < 0) + DPRINTF(E_LOG, L_DB, "Error fetching item by pos: pos (%d) relative to item with id (%d)\n", pos, item_id); + else + DPRINTF(E_DBG, L_DB, "Fetch by pos: fetched item (id=%d, pos=%d, file-id=%d)\n", queue_item->id, queue_item->pos, queue_item->file_id); return ret; } diff --git a/src/httpd_dacp.c b/src/httpd_dacp.c index f290e5f3..b3bad157 100644 --- a/src/httpd_dacp.c +++ b/src/httpd_dacp.c @@ -1017,6 +1017,8 @@ dacp_reply_cue_play(struct evhttp_request *req, struct evbuffer *evbuf, char **u } } + player_get_status(&status); + cuequery = evhttp_find_header(query, "query"); if (cuequery) { @@ -1031,12 +1033,9 @@ dacp_reply_cue_play(struct evhttp_request *req, struct evbuffer *evbuf, char **u return; } } - else + else if (status.status != PLAY_STOPPED) { - player_get_status(&status); - - if (status.status != PLAY_STOPPED) - player_playback_stop(); + player_playback_stop(); } param = evhttp_find_header(query, "dacp.shufflestate"); @@ -1597,6 +1596,8 @@ dacp_reply_playqueuecontents(struct evhttp_request *req, struct evbuffer *evbuf, return; } + player_get_status(&status); + /* * If the span parameter is negativ make song list for Previously Played, * otherwise make song list for Up Next and begin with first song after playlist position. @@ -1626,8 +1627,6 @@ dacp_reply_playqueuecontents(struct evhttp_request *req, struct evbuffer *evbuf, } else { - player_get_status(&status); - memset(&query_params, 0, sizeof(struct query_params)); if (status.shuffle) query_params.sort = S_SHUFFLE_POS; From 62dd1bb407146a90c3df58cd1cf52c5be296d13b Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 21 Jan 2017 10:13:40 +0100 Subject: [PATCH 32/33] [travic-ci] Added configuration file for travis-ci (#339) Builds different configuration with scan-build reporting failure if scan-build found issues. Due to the ANTLR3 generated files the checker deadcode.DeadStores needs to be disabled or all builds will fail. --- .travis.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..c1d175c7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +language: c +sudo: required +dist: trusty +env: + matrix: + - CFG="" + - CFG="--enable-lastfm" + - CFG="--enable-itunes" + - CFG="--enable-spotify" + +script: + - autoreconf -fi + - ./configure $CFG + - scan-build --status-bugs -disable-checker deadcode.DeadStores make + +before_install: + - wget -q -O - https://apt.mopidy.com/mopidy.gpg | sudo apt-key add - + - sudo wget -q -O /etc/apt/sources.list.d/mopidy.list https://apt.mopidy.com/jessie.list + - sudo apt-get -qq update + - sudo apt-get install -y build-essential clang git autotools-dev autoconf libtool gettext gawk gperf antlr3 libantlr3c-dev libconfuse-dev libunistring-dev libsqlite3-dev libavcodec-dev libavformat-dev libavfilter-dev libswscale-dev libavutil-dev libasound2-dev libmxml-dev libgcrypt11-dev libavahi-client-dev zlib1g-dev libevent-dev libplist-dev libcurl4-openssl-dev libjson-c-dev libspotify-dev + +# Disable email notification +notifications: + email: false \ No newline at end of file From e092a9ff3e4beb91f87cc94d92a2f8035126d1e0 Mon Sep 17 00:00:00 2001 From: chme Date: Sat, 21 Jan 2017 10:17:33 +0100 Subject: [PATCH 33/33] [spotify] Respect settings for 'artist_override' and 'album_override' (#340) when scanning spotify through the web api The goal is to reduce the artist/album cluttering that happens because of the users spotify playlists. If 'artist_override' is true, all tracks not in a saved album are treated as part of a compilation giving them the configured album-artist (e. g. "Various artists"). I 'album_override' is true, tracks not in a saved album will use the playlistname as albumname. --- src/spotify.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/spotify.c b/src/spotify.c index 70df5a3c..9fd05c47 100644 --- a/src/spotify.c +++ b/src/spotify.c @@ -2241,6 +2241,9 @@ scan_saved_albums() static int scan_playlisttracks(struct spotify_playlist *playlist, int plid) { + cfg_t *spotify_cfg; + bool artist_override; + bool album_override; struct spotify_request request; struct spotify_track track; struct media_file_info mfi; @@ -2248,6 +2251,10 @@ scan_playlisttracks(struct spotify_playlist *playlist, int plid) memset(&request, 0, sizeof(struct spotify_request)); + spotify_cfg = cfg_getsec(cfg, "spotify"); + artist_override = cfg_getbool(spotify_cfg, "artist_override"); + album_override = cfg_getbool(spotify_cfg, "album_override"); + while (0 == spotifywebapi_request_next(&request, playlist->tracks_href)) { db_transaction_begin(); @@ -2264,6 +2271,13 @@ scan_playlisttracks(struct spotify_playlist *playlist, int plid) memset(&mfi, 0, sizeof(struct media_file_info)); map_track_to_mfi(&track, &mfi); + track.is_compilation = (track.is_compilation || artist_override); + if (album_override) + { + free(mfi.album); + mfi.album = strdup(playlist->name); + } + library_process_media(track.uri, 1 /* TODO passing one prevents overwriting existing entries */, 0, DATA_KIND_SPOTIFY, 0, track.is_compilation, &mfi, dir_id); spotify_uri_register(track.uri);