diff --git a/README_JSON_API.md b/README_JSON_API.md index d9c37d98..01da66fa 100644 --- a/README_JSON_API.md +++ b/README_JSON_API.md @@ -746,6 +746,7 @@ curl -X PUT "http://localhost:3689/api/queue/items/2" | GET | [/api/library](#library-information) | Get library information | | GET | [/api/library/playlists](#list-playlists) | Get a list of playlists | | GET | [/api/library/playlists/{id}](#get-a-playlist) | Get a playlist | +| DELETE | [/api/library/playlists/{id}](#delete-a-playlist) | Delete a playlist | | GET | [/api/library/playlists/{id}/tracks](#list-playlist-tracks) | Get list of tracks for a playlist | | PUT | [/api/library/playlists/{id}/tracks](#update-playlist-tracks) | Update play count of tracks for a playlist | | GET | [/api/library/playlists/{id}/playlists](#list-playlists-in-a-playlist-folder) | Get list of playlists for a playlist folder | @@ -760,6 +761,7 @@ curl -X PUT "http://localhost:3689/api/queue/items/2" | GET | [/api/library/genres](#list-genres) | Get list of genres | | GET | [/api/library/count](#get-count-of-tracks-artists-and-albums) | Get count of tracks, artists and albums | | GET | [/api/library/files](#list-local-directories) | Get list of directories in the local library | +| POST | [/api/library/add](#add-an-item-to-the-library) | Add an item to the library | | PUT | [/api/update](#trigger-rescan) | Trigger a library rescan | | PUT | [/api/rescan](#trigger-meta-rescan) | Trigger a library metadata rescan | diff --git a/configure.ac b/configure.ac index f5e9361b..c3cef858 100644 --- a/configure.ac +++ b/configure.ac @@ -113,10 +113,16 @@ FORK_FUNC_REQUIRE([COMMON], [GNU libunistring], [LIBUNISTRING], [unistring], FORK_MODULES_CHECK([FORKED], [ZLIB], [zlib], [deflate], [zlib.h]) FORK_MODULES_CHECK([FORKED], [CONFUSE], [libconfuse >= 3.0], [cfg_init], [confuse.h]) -FORK_MODULES_CHECK([FORKED], [MINIXML], [mxml], [mxmlNewElement], [mxml.h], - [AC_CHECK_FUNCS([mxmlGetOpaque] [mxmlGetText] [mxmlGetType] [mxmlGetFirstChild])]) -dnl SQLite3 requires extra checks +FORK_MODULES_CHECK([FORKED], [MINIXML], [mxml], + [mxmlNewElement], [mxml.h], + [dnl check for old versions which have a serious memleak + AC_CHECK_FUNCS([mxmlGetOpaque] [mxmlGetText] [mxmlGetType] [mxmlGetFirstChild]) + PKG_CHECK_EXISTS([libevent >= 2.11], [], + [AC_DEFINE([HAVE_MXML_OLD], 1, + [Define to 1 if you have libmxml < 2.11)])]) + ]) + FORK_MODULES_CHECK([COMMON], [SQLITE3], [sqlite3 >= 3.5.0], [sqlite3_initialize], [sqlite3.h], [dnl Check that SQLite3 has the unlock notify API built-in @@ -137,7 +143,6 @@ FORK_MODULES_CHECK([COMMON], [SQLITE3], [sqlite3 >= 3.5.0], [AC_MSG_RESULT([[runtime will tell]])]) ]) -dnl libevent2 requires version checks FORK_MODULES_CHECK([FORKED], [LIBEVENT], [libevent >= 2], [event_base_new], [event2/event.h], [dnl check for old version @@ -146,7 +151,6 @@ FORK_MODULES_CHECK([FORKED], [LIBEVENT], [libevent >= 2], [Define to 1 if you have libevent 2 (<2.1.4)])]) ]) -dnl json-c version checks FORK_MODULES_CHECK([FORKED], [JSON_C], [json-c], [json_tokener_parse], [json.h], [dnl check for old version @@ -155,7 +159,6 @@ FORK_MODULES_CHECK([FORKED], [JSON_C], [json-c], [Define to 1 if you have json-c < 0.11])]) ]) -dnl antlr version checks FORK_FUNC_REQUIRE([FORKED], [ANTLR3 C runtime], [ANTLR3C], [antlr3c], [antlr3BaseRecognizerNew], [antlr3.h], [AC_CHECK_FUNC([[antlr3NewAsciiStringInPlaceStream]], diff --git a/src/Makefile.am b/src/Makefile.am index 0ad66365..81c2c1be 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -103,6 +103,7 @@ forked_daapd_SOURCES = main.c \ library/filescanner.c library/filescanner.h \ library/filescanner_ffmpeg.c library/filescanner_playlist.c \ library/filescanner_smartpl.c $(ITUNES_SRC) \ + library/rssscanner.c \ library.c library.h \ $(MDNS_SRC) mdns.h \ remote_pairing.c remote_pairing.h \ diff --git a/src/db.c b/src/db.c index 32929c47..973d197b 100644 --- a/src/db.c +++ b/src/db.c @@ -511,7 +511,7 @@ db_data_kind_label(enum data_kind data_kind) } /* Keep in sync with enum pl_type */ -static char *pl_type_label[] = { "special", "folder", "smart", "plain" }; +static char *pl_type_label[] = { "special", "folder", "smart", "plain", "rss" }; const char * db_pl_type_label(enum pl_type pl_type) @@ -1876,6 +1876,7 @@ db_build_query_plitems(struct query_params *qp, struct query_clause *qc) query = db_build_query_plitems_smart(qp, pli); break; + case PL_RSS: case PL_PLAIN: case PL_FOLDER: query = db_build_query_plitems_plain(qp, qc); @@ -3631,18 +3632,44 @@ void db_pl_delete(int id) { #define Q_TMPL "DELETE FROM playlists WHERE id = %d;" +#define Q_ORPHAN "SELECT filepath FROM playlistitems WHERE filepath NOT IN (SELECT filepath FROM playlistitems WHERE playlistid <> %d) AND playlistid = %d" +#define Q_FILES "DELETE FROM files WHERE data_kind = %d AND path IN (" Q_ORPHAN ");" char *query; int ret; if (id == 1) return; + db_transaction_begin(); + query = sqlite3_mprintf(Q_TMPL, id); ret = db_query_run(query, 1, 0); + if (ret < 0) + { + db_transaction_rollback(); + return; + } - if (ret == 0) - db_pl_clear_items(id); + // Remove orphaned files (http items in files must have been added by the + // playlist. The GROUP BY/count makes sure the files are not referenced by any + // other playlist. + // TODO find a cleaner way of identifying tracks added by a playlist + query = sqlite3_mprintf(Q_FILES, DATA_KIND_HTTP, id, id); + + ret = db_query_run(query, 1, 0); + if (ret < 0) + { + db_transaction_rollback(); + return; + } + + // Clear playlistitems + db_pl_clear_items(id); + + db_transaction_end(); +#undef Q_FILES +#undef Q_ORPHAN #undef Q_TMPL } diff --git a/src/db.h b/src/db.h index 8186db2b..150f43b3 100644 --- a/src/db.h +++ b/src/db.h @@ -236,6 +236,7 @@ enum pl_type { PL_FOLDER = 1, PL_SMART = 2, PL_PLAIN = 3, + PL_RSS = 4, PL_MAX, }; @@ -255,8 +256,8 @@ struct playlist_info { char *virtual_path; /* virtual path of underlying playlist */ uint32_t parent_id; /* Id of parent playlist if the playlist is nested */ uint32_t directory_id; /* Id of directory */ - char *query_order; /* order by clause if it is a smart playlist */ - int32_t query_limit; /* limit if it is a smart playlist */ + char *query_order; /* order by clause, used by e.g. a smart playlists */ + int32_t query_limit; /* limit, used by e.g. smart playlists */ uint32_t media_kind; uint32_t items; /* number of items (mimc) */ uint32_t streams; /* number of internet streams */ diff --git a/src/httpd_daap.c b/src/httpd_daap.c index 1c6472e5..37721c40 100644 --- a/src/httpd_daap.c +++ b/src/httpd_daap.c @@ -481,7 +481,7 @@ user_agent_filter(struct query_params *qp, struct httpd_request *hreq) // contained extended_media_kind:1, which characterise the queries we want // to filter. TODO: Not a really nice way of doing this, but best I could // think of. - if (!qp->filter || !strstr(qp->filter, "f.media_kind")) + if (!qp->filter || !strstr(qp->filter, "f.media_kind = 1")) return; filter = safe_asprintf("%s AND (f.data_kind <> %d)", qp->filter, DATA_KIND_HTTP); diff --git a/src/httpd_jsonapi.c b/src/httpd_jsonapi.c index f24a61d4..c16f1329 100644 --- a/src/httpd_jsonapi.c +++ b/src/httpd_jsonapi.c @@ -3200,7 +3200,7 @@ jsonapi_reply_library_playlists(struct httpd_request *hreq) query_params.type = Q_PL; query_params.sort = S_PLAYLIST; - query_params.filter = db_mprintf("(f.type = %d OR f.type = %d)", PL_PLAIN, PL_SMART); + query_params.filter = db_mprintf("(f.type = %d OR f.type = %d OR f.type = %d)", PL_PLAIN, PL_SMART, PL_RSS); ret = fetch_playlists(&query_params, items, &total); free(query_params.filter); @@ -3331,6 +3331,25 @@ jsonapi_reply_library_playlist_tracks(struct httpd_request *hreq) return HTTP_OK; } +static int +jsonapi_reply_library_playlist_delete(struct httpd_request *hreq) +{ + uint32_t pl_id; + int ret; + + ret = safe_atou32(hreq->uri_parsed->path_parts[3], &pl_id); + if (ret < 0) + { + DPRINTF(E_LOG, L_WEB, "No valid playlist id given '%s'\n", hreq->uri_parsed->path); + + return HTTP_BADREQUEST; + } + + db_pl_delete(pl_id); + + return HTTP_NOCONTENT; +} + static int jsonapi_reply_library_playlist_playlists(struct httpd_request *hreq) { @@ -3365,8 +3384,8 @@ jsonapi_reply_library_playlist_playlists(struct httpd_request *hreq) query_params.type = Q_PL; query_params.sort = S_PLAYLIST; - query_params.filter = db_mprintf("f.parent_id = %d AND (f.type = %d OR f.type = %d OR f.type = %d)", - playlist_id, PL_PLAIN, PL_SMART, PL_FOLDER); + query_params.filter = db_mprintf("f.parent_id = %d AND (f.type = %d OR f.type = %d OR f.type = %d OR f.type = %d)", + playlist_id, PL_PLAIN, PL_SMART, PL_RSS, PL_FOLDER); ret = fetch_playlists(&query_params, items, &total); if (ret < 0) @@ -3690,6 +3709,26 @@ jsonapi_reply_library_files(struct httpd_request *hreq) return HTTP_OK; } +static int +jsonapi_reply_library_add(struct httpd_request *hreq) +{ + const char *url; + int ret; + + url = evhttp_find_header(hreq->query, "url"); + if (!url) + { + DPRINTF(E_LOG, L_WEB, "Missing URL parameter for library add\n"); + return HTTP_BADREQUEST; + } + + ret = library_item_add(url); + if (ret < 0) + return HTTP_INTERNAL; + + return HTTP_OK; +} + static int search_tracks(json_object *reply, struct httpd_request *hreq, const char *param_query, struct smartpl *smartpl_expression, enum media_kind media_kind) { @@ -3896,7 +3935,7 @@ search_playlists(json_object *reply, struct httpd_request *hreq, const char *par query_params.type = Q_PL; query_params.sort = S_PLAYLIST; - query_params.filter = db_mprintf("((f.type = %d OR f.type = %d) AND f.title LIKE '%%%q%%')", PL_PLAIN, PL_SMART, param_query); + query_params.filter = db_mprintf("((f.type = %d OR f.type = %d OR f.type = %d) AND f.title LIKE '%%%q%%')", PL_PLAIN, PL_SMART, PL_RSS, param_query); ret = fetch_playlists(&query_params, items, &total); if (ret < 0) @@ -4059,7 +4098,7 @@ static struct httpd_uri_map adm_handlers[] = { EVHTTP_REQ_GET, "^/api/library/playlists/[[:digit:]]+/tracks$", jsonapi_reply_library_playlist_tracks }, { EVHTTP_REQ_PUT, "^/api/library/playlists/[[:digit:]]+/tracks", jsonapi_reply_library_playlist_tracks_put_byid}, // { EVHTTP_REQ_POST, "^/api/library/playlists/[[:digit:]]+/tracks$", jsonapi_reply_library_playlists_tracks }, -// { EVHTTP_REQ_DELETE, "^/api/library/playlists/[[:digit:]]+$", jsonapi_reply_library_playlist_tracks }, + { EVHTTP_REQ_DELETE, "^/api/library/playlists/[[:digit:]]+$", jsonapi_reply_library_playlist_delete }, { EVHTTP_REQ_GET, "^/api/library/playlists/[[:digit:]]+/playlists", jsonapi_reply_library_playlist_playlists }, { EVHTTP_REQ_GET, "^/api/library/artists$", jsonapi_reply_library_artists }, { EVHTTP_REQ_GET, "^/api/library/artists/[[:digit:]]+$", jsonapi_reply_library_artist }, @@ -4073,6 +4112,7 @@ static struct httpd_uri_map adm_handlers[] = { EVHTTP_REQ_GET, "^/api/library/genres$", jsonapi_reply_library_genres}, { EVHTTP_REQ_GET, "^/api/library/count$", jsonapi_reply_library_count }, { EVHTTP_REQ_GET, "^/api/library/files$", jsonapi_reply_library_files }, + { EVHTTP_REQ_POST, "^/api/library/add$", jsonapi_reply_library_add }, { EVHTTP_REQ_GET, "^/api/search$", jsonapi_reply_search }, diff --git a/src/httpd_rsp.c b/src/httpd_rsp.c index fb16fd32..17c5fa58 100644 --- a/src/httpd_rsp.c +++ b/src/httpd_rsp.c @@ -30,7 +30,7 @@ #include #include -#include +#include "mxml-compat.h" #include "httpd_rsp.h" #include "logger.h" diff --git a/src/inputs/pipe.c b/src/inputs/pipe.c index e8e18c69..62f22c77 100644 --- a/src/inputs/pipe.c +++ b/src/inputs/pipe.c @@ -53,7 +53,8 @@ #include #include -#include + +#include "mxml-compat.h" #include "input.h" #include "misc.h" @@ -64,7 +65,6 @@ #include "player.h" #include "worker.h" #include "commands.h" -#include "mxml-compat.h" // Maximum number of pipes to watch for data #define PIPE_MAX_WATCH 4 diff --git a/src/lastfm.c b/src/lastfm.c index 52895ede..08c6253a 100644 --- a/src/lastfm.c +++ b/src/lastfm.c @@ -31,7 +31,6 @@ #include #include -#include #include #include diff --git a/src/library.c b/src/library.c index 4bbbb305..98d37f7d 100644 --- a/src/library.c +++ b/src/library.c @@ -48,6 +48,15 @@ #include "listener.h" #include "player.h" +#define LIBRARY_MAX_CALLBACKS 16 + +struct library_callback_register +{ + library_cb cb; + void *arg; + struct event *ev; +}; + struct playlist_item_add_param { const char *vp_playlist; @@ -73,12 +82,14 @@ extern struct library_source filescanner; #ifdef HAVE_SPOTIFY_H extern struct library_source spotifyscanner; #endif +extern struct library_source rssscanner; static struct library_source *sources[] = { &filescanner, #ifdef HAVE_SPOTIFY_H &spotifyscanner, #endif + &rssscanner, NULL }; @@ -105,6 +116,9 @@ static struct event *updateev; static unsigned int deferred_update_notifications; static short deferred_update_events; +// Stores callbacks that backends may have requested +static struct library_callback_register library_cb_register[LIBRARY_MAX_CALLBACKS]; + /* ------------------- CALLED BY LIBRARY SOURCE MODULES -------------------- */ @@ -153,6 +167,77 @@ library_playlist_save(struct playlist_info *pli) return db_pl_update(pli); } +static void +scheduled_cb(int fd, short what, void *arg) +{ + struct library_callback_register *cbreg = arg; + library_cb cb = cbreg->cb; + void *cb_arg = cbreg->arg; + + // Must reset the register before calling back, otherwise it won't work if the + // callback reschedules by calling library_callback_schedule() + event_free(cbreg->ev); + memset(cbreg, 0, sizeof(struct library_callback_register)); + + DPRINTF(E_DBG, L_LIB, "Executing library callback to %p\n", cb); + cb(cb_arg); +} + +int +library_callback_schedule(library_cb cb, void *arg, struct timeval *wait, enum library_cb_action action) +{ + struct library_callback_register *cbreg; + bool replace_done; + int idx_available; + int i; + + for (i = 0, idx_available = -1, replace_done = false; i < ARRAY_SIZE(library_cb_register); i++) + { + if (idx_available == -1 && library_cb_register[i].cb == NULL) + idx_available = i; + + if (library_cb_register[i].cb != cb) + continue; + + if (action == LIBRARY_CB_REPLACE || action == LIBRARY_CB_ADD_OR_REPLACE) + { + event_add(library_cb_register[i].ev, wait); + library_cb_register[i].arg = arg; + replace_done = true; + } + else if (action == LIBRARY_CB_DELETE) + { + event_free(library_cb_register[i].ev); + memset(&library_cb_register[i], 0, sizeof(struct library_callback_register)); + } + } + + if (action == LIBRARY_CB_REPLACE || action == LIBRARY_CB_DELETE || (action == LIBRARY_CB_ADD_OR_REPLACE && replace_done)) + { + return 0; // All done + } + else if (idx_available == -1) + { + DPRINTF(E_LOG, L_LIB, "Error scheduling callback, register full (size=%d, action=%d)\n", LIBRARY_MAX_CALLBACKS, action); + return -1; + } + + cbreg = &library_cb_register[idx_available]; + cbreg->cb = cb; + cbreg->arg = arg; + + if (!cbreg->ev) + cbreg->ev = evtimer_new(evbase_lib, scheduled_cb, cbreg); + + CHECK_NULL(L_LIB, cbreg->ev); + + event_add(cbreg->ev, wait); + + DPRINTF(E_DBG, L_LIB, "Added library callback to %p (id %d), wait %ld.%06ld\n", cbreg->cb, idx_available, wait->tv_sec, wait->tv_usec); + + return idx_available; +} + /* ---------------------- LIBRARY ABSTRACTION --------------------- */ /* thread: library */ @@ -286,7 +371,7 @@ fullrescan(void *arg, int *ret) player_playback_stop(); db_queue_clear(0); - db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups + db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups, incl RSS for (i = 0; sources[i]; i++) { @@ -441,6 +526,36 @@ queue_save(void *arg, int *retval) return COMMAND_END; } +static enum command_state +item_add(void *arg, int *retval) +{ + const char *path = arg; + int i; + int ret = LIBRARY_ERROR; + + DPRINTF(E_DBG, L_LIB, "Adding item to library '%s'\n", path); + + for (i = 0; sources[i]; i++) + { + if (sources[i]->disabled || !sources[i]->item_add) + { + DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support add_item\n", sources[i]->name); + continue; + } + + ret = sources[i]->item_add(path); + + if (ret == LIBRARY_OK) + { + DPRINTF(E_DBG, L_LIB, "Add item to path '%s' with library source '%s'\n", path, sources[i]->name); + listener_notify(LISTENER_DATABASE); + break; + } + } + + *retval = ret; + return COMMAND_END; +} // Callback to notify listeners of database changes static void @@ -644,6 +759,15 @@ library_queue_item_add(const char *path, int position, char reshuffle, uint32_t return commands_exec_sync(cmdbase, queue_item_add, NULL, ¶m); } +int +library_item_add(const char *path) +{ + if (library_is_scanning()) + return -1; + + return commands_exec_sync(cmdbase, item_add, NULL, (char *)path); +} + int library_exec_async(command_function func, void *arg) { @@ -706,21 +830,18 @@ library_init(void) for (i = 0; sources[i]; i++) { - if (!sources[i]->init) - { - DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' has no init()\n", sources[i]->name); - return -1; - } - if (!sources[i]->initscan || !sources[i]->rescan || !sources[i]->metarescan || !sources[i]->fullrescan) { DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' is missing a scanning method\n", sources[i]->name); return -1; } - ret = sources[i]->init(); - if (ret < 0) - sources[i]->disabled = 1; + if (sources[i]->init && !sources[i]->disabled) + { + ret = sources[i]->init(); + if (ret < 0) + sources[i]->disabled = 1; + } } CHECK_NULL(L_LIB, cmdbase = commands_base_new(evbase_lib, NULL)); @@ -757,7 +878,13 @@ library_deinit() for (i = 0; sources[i]; i++) { if (sources[i]->deinit && !sources[i]->disabled) - sources[i]->deinit(); + sources[i]->deinit(); + } + + for (i = 0; i < ARRAY_SIZE(library_cb_register); i++) + { + if (library_cb_register[i].ev) + event_free(library_cb_register[i].ev); } event_free(updateev); diff --git a/src/library.h b/src/library.h index cfe6b24c..0c0a661a 100644 --- a/src/library.h +++ b/src/library.h @@ -30,6 +30,23 @@ #define LIBRARY_ERROR -1 #define LIBRARY_PATH_INVALID -2 +typedef void (*library_cb)(void *arg); + +/* + * Argument to library_callback_schedule() + */ +enum library_cb_action +{ + // Add as new callback + LIBRARY_CB_ADD, + // Replace callback if it already exists + LIBRARY_CB_REPLACE, + // Replace callback if it already exists, otherwise add as new + LIBRARY_CB_ADD_OR_REPLACE, + // Delete a callback + LIBRARY_CB_DELETE, +}; + /* * Definition of a library source * @@ -71,6 +88,11 @@ struct library_source */ int (*fullrescan)(void); + /* + * Add an item to the library + */ + int (*item_add)(const char *path); + /* * Add item to playlist */ @@ -94,6 +116,12 @@ struct library_source /* --------------------- Interface towards source backends ----------------- */ +/* + * Adds a mfi if mfi->id == 0, otherwise updates. + * + * @param mfi Media to save + * @return 0 if operation succeeded, -1 on failure. + */ int library_media_save(struct media_file_info *mfi); @@ -101,11 +129,28 @@ library_media_save(struct media_file_info *mfi); * Adds a playlist if pli->id == 0, otherwise updates. * * @param pli Playlist to save - * @return playlist id if operation succeeded, -1 on failure. + * @return Playlist id if operation succeeded, -1 on failure. */ int library_playlist_save(struct playlist_info *pli); +/* + * @param cb Callback to call + * @param arg Argument to call back with + * @param timeval How long to wait before calling back + * @param action (see enum) + * @return id of the scheduled event, -1 on failure + */ +int +library_callback_schedule(library_cb cb, void *arg, struct timeval *wait, enum library_cb_action action); + +/* + * @return true if a running scan should be aborted due to imminent shutdown + */ +bool +library_is_exiting(); + + /* ------------------------ Library external interface --------------------- */ void @@ -129,12 +174,6 @@ library_is_scanning(); void library_set_scanning(bool is_scanning); -/* - * @return true if a running scan should be aborted due to imminent shutdown, otherwise false - */ -bool -library_is_exiting(); - /* * Trigger for sending the DATABASE event * @@ -156,6 +195,10 @@ library_queue_save(char *path); int library_queue_item_add(const char *path, int position, char reshuffle, uint32_t item_id, int *count, int *new_item_id); +int +library_item_add(const char *path); + + /* * Execute the function 'func' with the given argument 'arg' in the library thread. * diff --git a/src/library/rssscanner.c b/src/library/rssscanner.c new file mode 100644 index 00000000..a7a0ec8a --- /dev/null +++ b/src/library/rssscanner.c @@ -0,0 +1,618 @@ +/* + * Copyright (C) 2020 whatdoineed2d/Ray + * based heavily on filescanner_playlist.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// For strptime() +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#endif +#include + +#include + +#include "mxml-compat.h" + +#include "conffile.h" +#include "logger.h" +#include "db.h" +#include "http.h" +#include "misc.h" +#include "misc_json.h" +#include "library.h" +#include "library/filescanner.h" + +#define APPLE_PODCASTS_SERVER "https://podcasts.apple.com/" +#define APPLE_ITUNES_SERVER "https://itunes.apple.com/" +#define RSS_LIMIT_DEFAULT 10 + +enum rss_scan_type { + RSS_SCAN_RESCAN, + RSS_SCAN_META, +}; + +struct rss_item_info { + const char *title; + const char *pubdate; + const char *link; + const char *url; + const char *type; +}; + +static struct timeval rss_refresh_interval = { 3600, 0 }; + +// Forward +static void +rss_refresh(void *arg); + +// RSS spec: https://validator.w3.org/feed/docs/rss2.html +static void +rss_date(struct tm *tm, const char *date) +{ + // RFC822 https://tools.ietf.org/html/rfc822#section-5 + // ie Fri, 07 Feb 2020 18:58:00 +0000 + // ^^^^ ^^^^^ + // optional ^^^^^ + // could also be GMT/UT/EST/A..I/M..Z + + const char *ptr; + time_t t; + + memset(tm, 0, sizeof(struct tm)); + ptr = strptime(date, "%a,%n", tm); // Looks for optional day of week + if (!ptr) + ptr = date; + + ptr = strptime(ptr, "%d%n%b%n%Y%n%H:%M:%S%n", tm); + if (!ptr) + { + // date is junk, using current time + time(&t); + gmtime_r(&t, tm); + } + + // TODO - adjust the TZ? +} + +// Makes a request to Apple based on the Apple Podcast ID in rss_url. The JSON +// response is parsed to find the original feed's url. Example rss_url: +// https://podcasts.apple.com/is/podcast/cgp-grey/id974722423 +static char * +apple_rss_feedurl_get(const char *rss_url) +{ + struct http_client_ctx ctx; + struct evbuffer *evbuf; + char url[100]; + const char *ptr; + unsigned podcast_id; + json_object *jresponse; + json_object *jfeedurl; + char *feedurl; + int ret; + + ptr = strrchr(rss_url, '/'); + if (!ptr) + { + DPRINTF(E_LOG, L_LIB, "Could not parse Apple Podcast RSS ID from '%s'\n", rss_url); + return NULL; + } + + ret = sscanf(ptr, "/id%u", &podcast_id); + if (ret != 1) + { + DPRINTF(E_LOG, L_LIB, "Could not parse Apple Podcast RSS ID from '%s'\n", rss_url); + return NULL; + } + + CHECK_NULL(L_LIB, evbuf = evbuffer_new()); + snprintf(url, sizeof(url), "%slookup?id=%u", APPLE_ITUNES_SERVER, podcast_id); + + memset(&ctx, 0, sizeof(struct http_client_ctx)); + ctx.url = url; + ctx.input_body = evbuf; + + ret = http_client_request(&ctx); + if (ret < 0 || ctx.response_code != HTTP_OK) + { + evbuffer_free(evbuf); + return NULL; + } + + jresponse = jparse_obj_from_evbuffer(evbuf); + evbuffer_free(evbuf); + if (!jresponse) + { + DPRINTF(E_LOG, L_LIB, "Could not parse RSS Apple response, podcast id %u\n", podcast_id); + return NULL; + } + + /* expect json resp - get feedUrl + * { + * "resultCount": 1, + * "results": [ + * { + * "wrapperType": "track", + * "kind": "podcast", + * ... + * "collectionViewUrl": "https://podcasts.apple.com/us/podcast/cgp-grey/id974722423?uo=4", + * "feedUrl": "http://cgpgrey.libsyn.com/rss", + * ... + * "genres": [ + * "Education", + * "Podcasts", + * "News" + * ] + * } + * ] + *} + */ + jfeedurl = JPARSE_SELECT(jresponse, "results", "feedUrl"); + if (!jfeedurl || json_object_get_type(jfeedurl) != json_type_string) + { + DPRINTF(E_LOG, L_LIB, "Could not find RSS feedUrl in response from Apple, podcast id %u\n", podcast_id); + jparse_free(jresponse); + return NULL; + } + + feedurl = safe_strdup(json_object_get_string(jfeedurl)); + + DPRINTF(E_DBG, L_LIB, "Mapped Apple podcast URL: '%s' -> '%s'\n", rss_url, feedurl); + + jparse_free(jresponse); + return feedurl; +} + +static struct playlist_info * +playlist_fetch(bool *is_new, const char *path) +{ + struct playlist_info *pli; + int ret; + + pli = db_pl_fetch_bypath(path); + if (pli) + { + db_pl_clear_items(pli->id); + *is_new = false; + return pli; + } + + CHECK_NULL(L_SCAN, pli = calloc(1, sizeof(struct playlist_info))); + + ret = playlist_fill(pli, path); + if (ret < 0) + goto error; + + pli->directory_id = DIR_HTTP; + pli->type = PL_RSS; + pli->query_limit = RSS_LIMIT_DEFAULT; + + ret = library_playlist_save(pli); + if (ret < 0) + goto error; + + pli->id = ret; + *is_new = true; + return pli; + + error: + DPRINTF(E_LOG, L_SCAN, "Error adding playlist for RSS feed '%s'\n", path); + free_pli(pli, 0); + return NULL; +} + +static mxml_node_t * +rss_xml_get(const char *url) +{ + struct http_client_ctx ctx = { 0 }; + const char *raw = NULL; + mxml_node_t *xml = NULL; + char *feedurl; + int ret; + + // Is it an apple podcast stream? + // ie https://podcasts.apple.com/is/podcast/cgp-grey/id974722423 + if (strncmp(url, APPLE_PODCASTS_SERVER, strlen(APPLE_PODCASTS_SERVER)) == 0) + { + feedurl = apple_rss_feedurl_get(url); + if (!feedurl) + return NULL; + } + else + feedurl = strdup(url); + + CHECK_NULL(L_LIB, ctx.input_body = evbuffer_new()); + ctx.url = feedurl; + + ret = http_client_request(&ctx); + if (ret < 0 || ctx.response_code != HTTP_OK) + { + DPRINTF(E_LOG, L_LIB, "Failed to fetch RSS from '%s' (return %d, error code %d)\n", ctx.url, ret, ctx.response_code); + goto cleanup; + } + + evbuffer_add(ctx.input_body, "", 1); + + raw = (const char*)evbuffer_pullup(ctx.input_body, -1); + + xml = mxmlLoadString(NULL, raw, MXML_OPAQUE_CALLBACK); + if (!xml) + { + DPRINTF(E_LOG, L_LIB, "Failed to parse RSS XML from '%s'\n", ctx.url); + goto cleanup; + } + + cleanup: + evbuffer_free(ctx.input_body); + free(feedurl); + return xml; +} + +static int +rss_xml_parse_feed(const char **feed_title, const char **feed_author, mxml_node_t *xml) +{ + mxml_node_t *channel; + mxml_node_t *node; + + channel = mxmlFindElement(xml, xml, "channel", NULL, NULL, MXML_DESCEND); + if (!channel) + { + DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml, missing 'channel' node\n"); + return -1; + } + + node = mxmlFindElement(channel, channel, "title", NULL, NULL, MXML_DESCEND_FIRST); + if (!node) + { + DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml, missing 'title' node\n"); + return -1; + } + *feed_title = mxmlGetOpaque(node); + + node = mxmlFindElement(channel, channel, "itunes:author", NULL, NULL, MXML_DESCEND_FIRST); + *feed_author = node ? mxmlGetOpaque(node) : NULL; + + return 0; +} + +static int +rss_xml_parse_item(struct rss_item_info *ri, mxml_node_t *xml, void **saveptr) +{ + mxml_node_t *item; + mxml_node_t *node; + const char *s; + + if (*saveptr) + { + item = (mxml_node_t *)(*saveptr); + while ( (item = mxmlGetNextSibling(item)) ) + { + s = mxmlGetElement(item); + if (s && strcmp(s, "item") == 0) + break; + } + *saveptr = item; + } + else + { + item = mxmlFindElement(xml, xml, "item", NULL, NULL, MXML_DESCEND); + *saveptr = item; + } + + if (!item) + return -1; // No more items + + memset(ri, 0, sizeof(struct rss_item_info)); + + node = mxmlFindElement(item, item, "title", NULL, NULL, MXML_DESCEND_FIRST); + ri->title = mxmlGetOpaque(node); + + node = mxmlFindElement(item, item, "pubDate", NULL, NULL, MXML_DESCEND_FIRST); + ri->pubdate = mxmlGetOpaque(node); + + node = mxmlFindElement(item, item, "link", NULL, NULL, MXML_DESCEND_FIRST); + ri->link = mxmlGetOpaque(node); + + node = mxmlFindElement(item, item, "enclosure", NULL, NULL, MXML_DESCEND_FIRST); + ri->url = mxmlElementGetAttr(node, "url"); + ri->type = mxmlElementGetAttr(node, "type"); + + return 0; +} + +static void +mfi_metadata_fixup(struct media_file_info *mfi, struct rss_item_info *ri, const char *feed_title, const char *feed_author, uint32_t time_added) +{ + struct tm tm; + + // Always take the meta from media file if possible; some podcasts (Apple) can + // use mp4 streams which tend not to have decent tags so in those cases take + // info from the RSS and not the stream + if (!mfi->artist) + mfi->artist = safe_strdup(feed_author); + if (!mfi->album) + mfi->album = safe_strdup(feed_title); + if (!mfi->url) + mfi->url = safe_strdup(ri->link); + if (!mfi->genre || strcmp("(186)Podcast", mfi->genre) == 0) + { + free(mfi->genre); + mfi->genre = strdup("Podcast"); + } + + // The title from the xml is usually better quality + if (ri->title) + { + free(mfi->title); + mfi->title = strdup(ri->title); + } + + // Remove, some can be very verbose + free(mfi->comment); + mfi->comment = NULL; + + // Date is always from the RSS feed info + rss_date(&tm, ri->pubdate); + mfi->date_released = mktime(&tm); + mfi->year = 1900 + tm.tm_year; + + mfi->media_kind = MEDIA_KIND_PODCAST; + + mfi->time_added = time_added; +} + +static int +rss_save(struct playlist_info *pli, int *count, enum rss_scan_type scan_type) +{ + mxml_node_t *xml; + const char *feed_title; + const char *feed_author; + struct media_file_info mfi = { 0 }; + struct rss_item_info ri; + uint32_t time_added; + void *ptr = NULL; + int ret; + + xml = rss_xml_get(pli->path); + if (!xml) + { + DPRINTF(E_LOG, L_LIB, "Could not get RSS/xml from '%s' (id %d)\n", pli->path, pli->id); + return -1; + } + + ret = rss_xml_parse_feed(&feed_title, &feed_author, xml); + if (ret < 0) + { + DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml received from '%s' (id %d)\n", pli->path, pli->id); + mxmlDelete(xml); + return -1; + } + + free(pli->title); + pli->title = safe_strdup(feed_title); + + // Fake the time - useful when we are adding a new stream - since the + // newest podcasts are added first (the stream is most recent first) + // having time_added date which is older on the most recent episodes + // makes no sense so make all the dates the same for a singleu update + time_added = (uint32_t)time(NULL); + + // Walk through the xml, saving each item + *count = 0; + db_transaction_begin(); + while ((ret = rss_xml_parse_item(&ri, xml, &ptr)) == 0 && (*count < pli->query_limit)) + { + if (library_is_exiting()) + { + db_transaction_rollback(); + mxmlDelete(xml); + return -1; + } + + if (!ri.url) + { + DPRINTF(E_WARN, L_LIB, "Missing URL for item '%s' (date %s) in RSS feed '%s'\n", ri.title, ri.pubdate, feed_title); + continue; + } + + db_pl_add_item_bypath(pli->id, ri.url); + (*count)++; + + // Try to just ping if already in library + if (scan_type == RSS_SCAN_RESCAN) + { + ret = db_file_ping_bypath(ri.url, 0); + if (ret > 0) + continue; + } + + scan_metadata_stream(&mfi, ri.url); + + mfi_metadata_fixup(&mfi, &ri, feed_title, feed_author, time_added); + + library_media_save(&mfi); + + free_mfi(&mfi, 1); + } + + db_transaction_end(); + mxmlDelete(xml); + + return 0; +} + +static int +rss_scan(const char *path, enum rss_scan_type scan_type) +{ + struct playlist_info *pli; + bool pl_is_new; + int count; + int ret; + + // Fetches or creates playlist, clears playlistitems + pli = playlist_fetch(&pl_is_new, path); + if (!pli) + return -1; + + // Retrieves the RSS and reads the feed, saving each item as a track, and also + // adds the relationship to playlistitems. The pli will also be updated with + // metadata from the RSS. + ret = rss_save(pli, &count, scan_type); + if (ret < 0) + goto error; + + // Save the playlist again, title etc may have been modified by rss_save(). + // This also updates the db_timestamp which protects the RSS from deletion. + ret = library_playlist_save(pli); + if (ret < 0) + goto error; + + DPRINTF(E_INFO, L_SCAN, "Added or updated %d items from RSS feed '%s' (id %d)\n", count, path, pli->id); + + free_pli(pli, 0); + return 0; + + error: + if (pl_is_new) + db_pl_delete(pli->id); + free_pli(pli, 0); + return -1; +} + +static void +rss_scan_all(enum rss_scan_type scan_type) +{ + struct query_params qp = { 0 }; + struct db_playlist_info dbpli; + time_t start; + time_t end; + int count; + int ret; + + DPRINTF(E_DBG, L_LIB, "Refreshing RSS feeds\n"); + + start = time(NULL); + + qp.type = Q_PL; + qp.sort = S_PLAYLIST; + qp.filter = db_mprintf("(f.type = %d)", PL_RSS); + + ret = db_query_start(&qp); + if (ret < 0) + { + DPRINTF(E_LOG, L_LIB, "Failed to find current RSS feeds from db\n"); + free(qp.filter); + return; + } + + count = 0; + while (((ret = db_query_fetch_pl(&qp, &dbpli)) == 0) && (dbpli.path)) + { + ret = rss_scan(dbpli.path, scan_type); + if (ret == 0) + count++; + } + + db_query_end(&qp); + free(qp.filter); + + end = time(NULL); + + if (count == 0) + return; + + library_callback_schedule(rss_refresh, NULL, &rss_refresh_interval, LIBRARY_CB_ADD_OR_REPLACE); + + DPRINTF(E_INFO, L_LIB, "Refreshed %d RSS feeds in %.f sec (scan type %d)\n", count, difftime(end, start), scan_type); +} + +static void +rss_refresh(void *arg) +{ + rss_scan_all(RSS_SCAN_RESCAN); +} + +static int +rss_rescan(void) +{ + rss_scan_all(RSS_SCAN_RESCAN); + + return LIBRARY_OK; +} + +static int +rss_metascan(void) +{ + rss_scan_all(RSS_SCAN_META); + + return LIBRARY_OK; +} + +static int +rss_fullscan(void) +{ + DPRINTF(E_LOG, L_LIB, "RSS feeds removed during full-rescan\n"); + + return LIBRARY_OK; +} + +static int +rss_add(const char *path) +{ + int ret; + + if (strncmp(path, "http://", 7) != 0 && strncmp(path, "https://", 8) != 0) + { + DPRINTF(E_SPAM, L_LIB, "Invalid RSS path '%s'\n", path); + return LIBRARY_PATH_INVALID; + } + + DPRINTF(E_DBG, L_LIB, "Adding RSS '%s'\n", path); + + ret = rss_scan(path, RSS_SCAN_RESCAN); + if (ret < 0) + return LIBRARY_PATH_INVALID; + + library_callback_schedule(rss_refresh, NULL, &rss_refresh_interval, LIBRARY_CB_ADD_OR_REPLACE); + + return LIBRARY_OK; +} + +struct library_source rssscanner = +{ + .name = "RSS feeds", + .disabled = 0, + .initscan = rss_rescan, + .rescan = rss_rescan, + .metarescan = rss_metascan, + .fullrescan = rss_fullscan, + .item_add = rss_add, +}; diff --git a/src/mxml-compat.h b/src/mxml-compat.h index 1437ad89..1b56f4a6 100644 --- a/src/mxml-compat.h +++ b/src/mxml-compat.h @@ -1,6 +1,96 @@ #ifndef __MXML_COMPAT_H__ #define __MXML_COMPAT_H__ +// mxml 2.10 has a memory leak in mxmlDelete, see https://github.com/michaelrsweet/mxml/issues/183 +// - and since this is the version in Ubuntu 18.04 LTS and Raspian Stretch, we +// fix it by including a fixed mxmlDelete here. It should be removed once the +// major distros no longer have 2.10. The below code is msweet's fixed mxml. +#ifndef HAVE_MXML_OLD +# include +#else +// Trick to undefine mxml.h's mxmlDelete +#define mxmlDelete mxmlDelete_memleak +# include +#undef mxmlDelete + +static void +compat_mxml_free(mxml_node_t *node) +{ + int i; + + switch (node->type) + { + case MXML_ELEMENT : + if (node->value.element.name) + free(node->value.element.name); + + if (node->value.element.num_attrs) + { + for (i = 0; i < node->value.element.num_attrs; i ++) + { + if (node->value.element.attrs[i].name) + free(node->value.element.attrs[i].name); + if (node->value.element.attrs[i].value) + free(node->value.element.attrs[i].value); + } + + free(node->value.element.attrs); + } + break; + case MXML_INTEGER : + break; + case MXML_OPAQUE : + if (node->value.opaque) + free(node->value.opaque); + break; + case MXML_REAL : + break; + case MXML_TEXT : + if (node->value.text.string) + free(node->value.text.string); + break; + case MXML_CUSTOM : + if (node->value.custom.data && + node->value.custom.destroy) + (*(node->value.custom.destroy))(node->value.custom.data); + break; + default : + break; + } + + free(node); +} + +static void +mxmlDelete(mxml_node_t *node) +{ + mxml_node_t *current, + *next; + + if (!node) + return; + + mxmlRemove(node); + for (current = node->child; current; current = next) + { + if ((next = current->child) != NULL) + { + current->child = NULL; + continue; + } + + if ((next = current->next) == NULL) + { + if ((next = current->parent) == node) + next = NULL; + } + compat_mxml_free(current); + } + + compat_mxml_free(node); +} +#endif + /* For compability with mxml 2.6 */ #ifndef HAVE_MXMLGETTEXT __attribute__((unused)) static const char * /* O - Text string or NULL */