MyFavMirrors
/
owntone-server
mirror of https://github.com/owntone/owntone-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'whatdoineed2do-rss1'

This commit is contained in:
ejurgensen 2020-04-04 21:26:16 +02:00
parent 9bb9576849 4fcd0ba781
commit 04d5246cc9
14 changed files with 990 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README_JSON_API.md
View File

@ -746,6 +746,7 @@ curl -X PUT "http://localhost:3689/api/queue/items/2"
| GET | [/api/library](#library-information) | Get library information |
| GET | [/api/library/playlists](#list-playlists) | Get a list of playlists |
| GET | [/api/library/playlists/{id}](#get-a-playlist) | Get a playlist |
| DELETE | [/api/library/playlists/{id}](#delete-a-playlist) | Delete a playlist |
| GET | [/api/library/playlists/{id}/tracks](#list-playlist-tracks) | Get list of tracks for a playlist |
| PUT | [/api/library/playlists/{id}/tracks](#update-playlist-tracks) | Update play count of tracks for a playlist |
| GET | [/api/library/playlists/{id}/playlists](#list-playlists-in-a-playlist-folder) | Get list of playlists for a playlist folder |
@ -760,6 +761,7 @@ curl -X PUT "http://localhost:3689/api/queue/items/2"
| GET | [/api/library/genres](#list-genres) | Get list of genres |
| GET | [/api/library/count](#get-count-of-tracks-artists-and-albums) | Get count of tracks, artists and albums |
| GET | [/api/library/files](#list-local-directories) | Get list of directories in the local library |
| POST | [/api/library/add](#add-an-item-to-the-library) | Add an item to the library |
| PUT | [/api/update](#trigger-rescan) | Trigger a library rescan |
| PUT | [/api/rescan](#trigger-meta-rescan) | Trigger a library metadata rescan |

15
configure.ac
View File

@ -113,10 +113,16 @@ FORK_FUNC_REQUIRE([COMMON], [GNU libunistring], [LIBUNISTRING], [unistring],
FORK_MODULES_CHECK([FORKED], [ZLIB], [zlib], [deflate], [zlib.h])
FORK_MODULES_CHECK([FORKED], [CONFUSE], [libconfuse >= 3.0], [cfg_init], [confuse.h])
FORK_MODULES_CHECK([FORKED], [MINIXML], [mxml], [mxmlNewElement], [mxml.h],
[AC_CHECK_FUNCS([mxmlGetOpaque] [mxmlGetText] [mxmlGetType] [mxmlGetFirstChild])])
dnl SQLite3 requires extra checks
FORK_MODULES_CHECK([FORKED], [MINIXML], [mxml],
[mxmlNewElement], [mxml.h],
[dnl check for old versions which have a serious memleak
AC_CHECK_FUNCS([mxmlGetOpaque] [mxmlGetText] [mxmlGetType] [mxmlGetFirstChild])
PKG_CHECK_EXISTS([libevent >= 2.11], [],
[AC_DEFINE([HAVE_MXML_OLD], 1,
[Define to 1 if you have libmxml < 2.11)])])
])
FORK_MODULES_CHECK([COMMON], [SQLITE3], [sqlite3 >= 3.5.0],
[sqlite3_initialize], [sqlite3.h],
[dnl Check that SQLite3 has the unlock notify API built-in
@ -137,7 +143,6 @@ FORK_MODULES_CHECK([COMMON], [SQLITE3], [sqlite3 >= 3.5.0],
[AC_MSG_RESULT([[runtime will tell]])])
])
dnl libevent2 requires version checks
FORK_MODULES_CHECK([FORKED], [LIBEVENT], [libevent >= 2],
[event_base_new], [event2/event.h],
[dnl check for old version
@ -146,7 +151,6 @@ FORK_MODULES_CHECK([FORKED], [LIBEVENT], [libevent >= 2],
[Define to 1 if you have libevent 2 (<2.1.4)])])
])
dnl json-c version checks
FORK_MODULES_CHECK([FORKED], [JSON_C], [json-c],
[json_tokener_parse], [json.h],
[dnl check for old version
@ -155,7 +159,6 @@ FORK_MODULES_CHECK([FORKED], [JSON_C], [json-c],
[Define to 1 if you have json-c < 0.11])])
])
dnl antlr version checks
FORK_FUNC_REQUIRE([FORKED], [ANTLR3 C runtime], [ANTLR3C], [antlr3c],
[antlr3BaseRecognizerNew], [antlr3.h],
[AC_CHECK_FUNC([[antlr3NewAsciiStringInPlaceStream]],

1
src/Makefile.am
View File

@ -103,6 +103,7 @@ forked_daapd_SOURCES = main.c \
library/filescanner.c library/filescanner.h \
library/filescanner_ffmpeg.c library/filescanner_playlist.c \
library/filescanner_smartpl.c $(ITUNES_SRC) \
library/rssscanner.c \
library.c library.h \
$(MDNS_SRC) mdns.h \
remote_pairing.c remote_pairing.h \

33
src/db.c
View File

@ -511,7 +511,7 @@ db_data_kind_label(enum data_kind data_kind)
}
/* Keep in sync with enum pl_type */
static char *pl_type_label[] = { "special", "folder", "smart", "plain" };
static char *pl_type_label[] = { "special", "folder", "smart", "plain", "rss" };
const char *
db_pl_type_label(enum pl_type pl_type)
@ -1876,6 +1876,7 @@ db_build_query_plitems(struct query_params *qp, struct query_clause *qc)
query = db_build_query_plitems_smart(qp, pli);
break;
case PL_RSS:
case PL_PLAIN:
case PL_FOLDER:
query = db_build_query_plitems_plain(qp, qc);
@ -3631,18 +3632,44 @@ void
db_pl_delete(int id)
{
#define Q_TMPL "DELETE FROM playlists WHERE id = %d;"
#define Q_ORPHAN "SELECT filepath FROM playlistitems WHERE filepath NOT IN (SELECT filepath FROM playlistitems WHERE playlistid <> %d) AND playlistid = %d"
#define Q_FILES "DELETE FROM files WHERE data_kind = %d AND path IN (" Q_ORPHAN ");"
char *query;
int ret;
if (id == 1)
return;
db_transaction_begin();
query = sqlite3_mprintf(Q_TMPL, id);
ret = db_query_run(query, 1, 0);
if (ret < 0)
{
db_transaction_rollback();
return;
}
if (ret == 0)
db_pl_clear_items(id);
// Remove orphaned files (http items in files must have been added by the
// playlist. The GROUP BY/count makes sure the files are not referenced by any
// other playlist.
// TODO find a cleaner way of identifying tracks added by a playlist
query = sqlite3_mprintf(Q_FILES, DATA_KIND_HTTP, id, id);
ret = db_query_run(query, 1, 0);
if (ret < 0)
{
db_transaction_rollback();
return;
}
// Clear playlistitems
db_pl_clear_items(id);
db_transaction_end();
#undef Q_FILES
#undef Q_ORPHAN
#undef Q_TMPL
}

5
src/db.h
View File

@ -236,6 +236,7 @@ enum pl_type {
PL_FOLDER = 1,
PL_SMART = 2,
PL_PLAIN = 3,
PL_RSS = 4,
PL_MAX,
};
@ -255,8 +256,8 @@ struct playlist_info {
char *virtual_path; /* virtual path of underlying playlist */
uint32_t parent_id; /* Id of parent playlist if the playlist is nested */
uint32_t directory_id; /* Id of directory */
char *query_order; /* order by clause if it is a smart playlist */
int32_t query_limit; /* limit if it is a smart playlist */
char *query_order; /* order by clause, used by e.g. a smart playlists */
int32_t query_limit; /* limit, used by e.g. smart playlists */
uint32_t media_kind;
uint32_t items; /* number of items (mimc) */
uint32_t streams; /* number of internet streams */

2
src/httpd_daap.c
View File

@ -481,7 +481,7 @@ user_agent_filter(struct query_params *qp, struct httpd_request *hreq)
// contained extended_media_kind:1, which characterise the queries we want
// to filter. TODO: Not a really nice way of doing this, but best I could
// think of.
if (!qp->filter || !strstr(qp->filter, "f.media_kind"))
if (!qp->filter || !strstr(qp->filter, "f.media_kind = 1"))
return;
filter = safe_asprintf("%s AND (f.data_kind <> %d)", qp->filter, DATA_KIND_HTTP);

50
src/httpd_jsonapi.c
View File

@ -3200,7 +3200,7 @@ jsonapi_reply_library_playlists(struct httpd_request *hreq)
query_params.type = Q_PL;
query_params.sort = S_PLAYLIST;
query_params.filter = db_mprintf("(f.type = %d OR f.type = %d)", PL_PLAIN, PL_SMART);
query_params.filter = db_mprintf("(f.type = %d OR f.type = %d OR f.type = %d)", PL_PLAIN, PL_SMART, PL_RSS);
ret = fetch_playlists(&query_params, items, &total);
free(query_params.filter);
@ -3331,6 +3331,25 @@ jsonapi_reply_library_playlist_tracks(struct httpd_request *hreq)
return HTTP_OK;
}
static int
jsonapi_reply_library_playlist_delete(struct httpd_request *hreq)
{
uint32_t pl_id;
int ret;
ret = safe_atou32(hreq->uri_parsed->path_parts[3], &pl_id);
if (ret < 0)
{
DPRINTF(E_LOG, L_WEB, "No valid playlist id given '%s'\n", hreq->uri_parsed->path);
return HTTP_BADREQUEST;
}
db_pl_delete(pl_id);
return HTTP_NOCONTENT;
}
static int
jsonapi_reply_library_playlist_playlists(struct httpd_request *hreq)
{
@ -3365,8 +3384,8 @@ jsonapi_reply_library_playlist_playlists(struct httpd_request *hreq)
query_params.type = Q_PL;
query_params.sort = S_PLAYLIST;
query_params.filter = db_mprintf("f.parent_id = %d AND (f.type = %d OR f.type = %d OR f.type = %d)",
playlist_id, PL_PLAIN, PL_SMART, PL_FOLDER);
query_params.filter = db_mprintf("f.parent_id = %d AND (f.type = %d OR f.type = %d OR f.type = %d OR f.type = %d)",
playlist_id, PL_PLAIN, PL_SMART, PL_RSS, PL_FOLDER);
ret = fetch_playlists(&query_params, items, &total);
if (ret < 0)
@ -3690,6 +3709,26 @@ jsonapi_reply_library_files(struct httpd_request *hreq)
return HTTP_OK;
}
static int
jsonapi_reply_library_add(struct httpd_request *hreq)
{
const char *url;
int ret;
url = evhttp_find_header(hreq->query, "url");
if (!url)
{
DPRINTF(E_LOG, L_WEB, "Missing URL parameter for library add\n");
return HTTP_BADREQUEST;
}
ret = library_item_add(url);
if (ret < 0)
return HTTP_INTERNAL;
return HTTP_OK;
}
static int
search_tracks(json_object *reply, struct httpd_request *hreq, const char *param_query, struct smartpl *smartpl_expression, enum media_kind media_kind)
{
@ -3896,7 +3935,7 @@ search_playlists(json_object *reply, struct httpd_request *hreq, const char *par
query_params.type = Q_PL;
query_params.sort = S_PLAYLIST;
query_params.filter = db_mprintf("((f.type = %d OR f.type = %d) AND f.title LIKE '%%%q%%')", PL_PLAIN, PL_SMART, param_query);
query_params.filter = db_mprintf("((f.type = %d OR f.type = %d OR f.type = %d) AND f.title LIKE '%%%q%%')", PL_PLAIN, PL_SMART, PL_RSS, param_query);
ret = fetch_playlists(&query_params, items, &total);
if (ret < 0)
@ -4059,7 +4098,7 @@ static struct httpd_uri_map adm_handlers[] =
{ EVHTTP_REQ_GET, "^/api/library/playlists/[[:digit:]]+/tracks$", jsonapi_reply_library_playlist_tracks },
{ EVHTTP_REQ_PUT, "^/api/library/playlists/[[:digit:]]+/tracks", jsonapi_reply_library_playlist_tracks_put_byid},
// { EVHTTP_REQ_POST, "^/api/library/playlists/[[:digit:]]+/tracks$", jsonapi_reply_library_playlists_tracks },
// { EVHTTP_REQ_DELETE, "^/api/library/playlists/[[:digit:]]+$", jsonapi_reply_library_playlist_tracks },
{ EVHTTP_REQ_DELETE, "^/api/library/playlists/[[:digit:]]+$", jsonapi_reply_library_playlist_delete },
{ EVHTTP_REQ_GET, "^/api/library/playlists/[[:digit:]]+/playlists", jsonapi_reply_library_playlist_playlists },
{ EVHTTP_REQ_GET, "^/api/library/artists$", jsonapi_reply_library_artists },
{ EVHTTP_REQ_GET, "^/api/library/artists/[[:digit:]]+$", jsonapi_reply_library_artist },
@ -4073,6 +4112,7 @@ static struct httpd_uri_map adm_handlers[] =
{ EVHTTP_REQ_GET, "^/api/library/genres$", jsonapi_reply_library_genres},
{ EVHTTP_REQ_GET, "^/api/library/count$", jsonapi_reply_library_count },
{ EVHTTP_REQ_GET, "^/api/library/files$", jsonapi_reply_library_files },
{ EVHTTP_REQ_POST, "^/api/library/add$", jsonapi_reply_library_add },
{ EVHTTP_REQ_GET, "^/api/search$", jsonapi_reply_search },

2
src/httpd_rsp.c
View File

@ -30,7 +30,7 @@
#include <sys/types.h>
#include <limits.h>
#include <mxml.h>
#include "mxml-compat.h"
#include "httpd_rsp.h"
#include "logger.h"

4
src/inputs/pipe.c
View File

@ -53,7 +53,8 @@
#include <event2/event.h>
#include <event2/buffer.h>
#include <mxml.h>
#include "mxml-compat.h"
#include "input.h"
#include "misc.h"
@ -64,7 +65,6 @@
#include "player.h"
#include "worker.h"
#include "commands.h"
#include "mxml-compat.h"
// Maximum number of pipes to watch for data
#define PIPE_MAX_WATCH 4

1
src/lastfm.c
View File

@ -31,7 +31,6 @@
#include <stdbool.h>
#include <gcrypt.h>
#include <mxml.h>
#include <event2/buffer.h>
#include <event2/http.h>

149
src/library.c
View File

@ -48,6 +48,15 @@
#include "listener.h"
#include "player.h"
#define LIBRARY_MAX_CALLBACKS 16
struct library_callback_register
{
library_cb cb;
void *arg;
struct event *ev;
};
struct playlist_item_add_param
{
const char *vp_playlist;
@ -73,12 +82,14 @@ extern struct library_source filescanner;
#ifdef HAVE_SPOTIFY_H
extern struct library_source spotifyscanner;
#endif
extern struct library_source rssscanner;
static struct library_source *sources[] = {
&filescanner,
#ifdef HAVE_SPOTIFY_H
&spotifyscanner,
#endif
&rssscanner,
NULL
};
@ -105,6 +116,9 @@ static struct event *updateev;
static unsigned int deferred_update_notifications;
static short deferred_update_events;
// Stores callbacks that backends may have requested
static struct library_callback_register library_cb_register[LIBRARY_MAX_CALLBACKS];
/* ------------------- CALLED BY LIBRARY SOURCE MODULES -------------------- */
@ -153,6 +167,77 @@ library_playlist_save(struct playlist_info *pli)
return db_pl_update(pli);
}
static void
scheduled_cb(int fd, short what, void *arg)
{
struct library_callback_register *cbreg = arg;
library_cb cb = cbreg->cb;
void *cb_arg = cbreg->arg;
// Must reset the register before calling back, otherwise it won't work if the
// callback reschedules by calling library_callback_schedule()
event_free(cbreg->ev);
memset(cbreg, 0, sizeof(struct library_callback_register));
DPRINTF(E_DBG, L_LIB, "Executing library callback to %p\n", cb);
cb(cb_arg);
}
int
library_callback_schedule(library_cb cb, void *arg, struct timeval *wait, enum library_cb_action action)
{
struct library_callback_register *cbreg;
bool replace_done;
int idx_available;
int i;
for (i = 0, idx_available = -1, replace_done = false; i < ARRAY_SIZE(library_cb_register); i++)
{
if (idx_available == -1 && library_cb_register[i].cb == NULL)
idx_available = i;
if (library_cb_register[i].cb != cb)
continue;
if (action == LIBRARY_CB_REPLACE || action == LIBRARY_CB_ADD_OR_REPLACE)
{
event_add(library_cb_register[i].ev, wait);
library_cb_register[i].arg = arg;
replace_done = true;
}
else if (action == LIBRARY_CB_DELETE)
{
event_free(library_cb_register[i].ev);
memset(&library_cb_register[i], 0, sizeof(struct library_callback_register));
}
}
if (action == LIBRARY_CB_REPLACE || action == LIBRARY_CB_DELETE || (action == LIBRARY_CB_ADD_OR_REPLACE && replace_done))
{
return 0; // All done
}
else if (idx_available == -1)
{
DPRINTF(E_LOG, L_LIB, "Error scheduling callback, register full (size=%d, action=%d)\n", LIBRARY_MAX_CALLBACKS, action);
return -1;
}
cbreg = &library_cb_register[idx_available];
cbreg->cb = cb;
cbreg->arg = arg;
if (!cbreg->ev)
cbreg->ev = evtimer_new(evbase_lib, scheduled_cb, cbreg);
CHECK_NULL(L_LIB, cbreg->ev);
event_add(cbreg->ev, wait);
DPRINTF(E_DBG, L_LIB, "Added library callback to %p (id %d), wait %ld.%06ld\n", cbreg->cb, idx_available, wait->tv_sec, wait->tv_usec);
return idx_available;
}
/* ---------------------- LIBRARY ABSTRACTION --------------------- */
/* thread: library */
@ -286,7 +371,7 @@ fullrescan(void *arg, int *ret)
player_playback_stop();
db_queue_clear(0);
db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups
db_purge_all(); // Clears files, playlists, playlistitems, inotify and groups, incl RSS
for (i = 0; sources[i]; i++)
{
@ -441,6 +526,36 @@ queue_save(void *arg, int *retval)
return COMMAND_END;
}
static enum command_state
item_add(void *arg, int *retval)
{
const char *path = arg;
int i;
int ret = LIBRARY_ERROR;
DPRINTF(E_DBG, L_LIB, "Adding item to library '%s'\n", path);
for (i = 0; sources[i]; i++)
{
if (sources[i]->disabled || !sources[i]->item_add)
{
DPRINTF(E_DBG, L_LIB, "Library source '%s' is disabled or does not support add_item\n", sources[i]->name);
continue;
}
ret = sources[i]->item_add(path);
if (ret == LIBRARY_OK)
{
DPRINTF(E_DBG, L_LIB, "Add item to path '%s' with library source '%s'\n", path, sources[i]->name);
listener_notify(LISTENER_DATABASE);
break;
}
}
*retval = ret;
return COMMAND_END;
}
// Callback to notify listeners of database changes
static void
@ -644,6 +759,15 @@ library_queue_item_add(const char *path, int position, char reshuffle, uint32_t
return commands_exec_sync(cmdbase, queue_item_add, NULL, &param);
}
int
library_item_add(const char *path)
{
if (library_is_scanning())
return -1;
return commands_exec_sync(cmdbase, item_add, NULL, (char *)path);
}
int
library_exec_async(command_function func, void *arg)
{
@ -706,21 +830,18 @@ library_init(void)
for (i = 0; sources[i]; i++)
{
if (!sources[i]->init)
{
DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' has no init()\n", sources[i]->name);
return -1;
}
if (!sources[i]->initscan || !sources[i]->rescan || !sources[i]->metarescan || !sources[i]->fullrescan)
{
DPRINTF(E_FATAL, L_LIB, "BUG: library source '%s' is missing a scanning method\n", sources[i]->name);
return -1;
}
ret = sources[i]->init();
if (ret < 0)
sources[i]->disabled = 1;
if (sources[i]->init && !sources[i]->disabled)
{
ret = sources[i]->init();
if (ret < 0)
sources[i]->disabled = 1;
}
}
CHECK_NULL(L_LIB, cmdbase = commands_base_new(evbase_lib, NULL));
@ -757,7 +878,13 @@ library_deinit()
for (i = 0; sources[i]; i++)
{
if (sources[i]->deinit && !sources[i]->disabled)
sources[i]->deinit();
sources[i]->deinit();
}
for (i = 0; i < ARRAY_SIZE(library_cb_register); i++)
{
if (library_cb_register[i].ev)
event_free(library_cb_register[i].ev);
}
event_free(updateev);

57
src/library.h
View File

@ -30,6 +30,23 @@
#define LIBRARY_ERROR -1
#define LIBRARY_PATH_INVALID -2
typedef void (*library_cb)(void *arg);
/*
* Argument to library_callback_schedule()
*/
enum library_cb_action
{
// Add as new callback
LIBRARY_CB_ADD,
// Replace callback if it already exists
LIBRARY_CB_REPLACE,
// Replace callback if it already exists, otherwise add as new
LIBRARY_CB_ADD_OR_REPLACE,
// Delete a callback
LIBRARY_CB_DELETE,
};
/*
* Definition of a library source
*
@ -71,6 +88,11 @@ struct library_source
*/
int (*fullrescan)(void);
/*
* Add an item to the library
*/
int (*item_add)(const char *path);
/*
* Add item to playlist
*/
@ -94,6 +116,12 @@ struct library_source
/* --------------------- Interface towards source backends ----------------- */
/*
* Adds a mfi if mfi->id == 0, otherwise updates.
*
* @param mfi Media to save
* @return 0 if operation succeeded, -1 on failure.
*/
int
library_media_save(struct media_file_info *mfi);
@ -101,11 +129,28 @@ library_media_save(struct media_file_info *mfi);
* Adds a playlist if pli->id == 0, otherwise updates.
*
* @param pli Playlist to save
* @return playlist id if operation succeeded, -1 on failure.
* @return Playlist id if operation succeeded, -1 on failure.
*/
int
library_playlist_save(struct playlist_info *pli);
/*
* @param cb Callback to call
* @param arg Argument to call back with
* @param timeval How long to wait before calling back
* @param action (see enum)
* @return id of the scheduled event, -1 on failure
*/
int
library_callback_schedule(library_cb cb, void *arg, struct timeval *wait, enum library_cb_action action);
/*
* @return true if a running scan should be aborted due to imminent shutdown
*/
bool
library_is_exiting();
/* ------------------------ Library external interface --------------------- */
void
@ -129,12 +174,6 @@ library_is_scanning();
void
library_set_scanning(bool is_scanning);
/*
* @return true if a running scan should be aborted due to imminent shutdown, otherwise false
*/
bool
library_is_exiting();
/*
* Trigger for sending the DATABASE event
*
@ -156,6 +195,10 @@ library_queue_save(char *path);
int
library_queue_item_add(const char *path, int position, char reshuffle, uint32_t item_id, int *count, int *new_item_id);
int
library_item_add(const char *path);
/*
* Execute the function 'func' with the given argument 'arg' in the library thread.
*

618
src/library/rssscanner.c Normal file
View File

@ -0,0 +1,618 @@
/*
* Copyright (C) 2020 whatdoineed2d/Ray
* based heavily on filescanner_playlist.c
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <stdio.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
// For strptime()
#ifndef _XOPEN_SOURCE
#define _XOPEN_SOURCE
#endif
#include <time.h>
#include <event2/buffer.h>
#include "mxml-compat.h"
#include "conffile.h"
#include "logger.h"
#include "db.h"
#include "http.h"
#include "misc.h"
#include "misc_json.h"
#include "library.h"
#include "library/filescanner.h"
#define APPLE_PODCASTS_SERVER "https://podcasts.apple.com/"
#define APPLE_ITUNES_SERVER "https://itunes.apple.com/"
#define RSS_LIMIT_DEFAULT 10
enum rss_scan_type {
RSS_SCAN_RESCAN,
RSS_SCAN_META,
};
struct rss_item_info {
const char *title;
const char *pubdate;
const char *link;
const char *url;
const char *type;
};
static struct timeval rss_refresh_interval = { 3600, 0 };
// Forward
static void
rss_refresh(void *arg);
// RSS spec: https://validator.w3.org/feed/docs/rss2.html
static void
rss_date(struct tm *tm, const char *date)
{
// RFC822 https://tools.ietf.org/html/rfc822#section-5
// ie Fri, 07 Feb 2020 18:58:00 +0000
// ^^^^ ^^^^^
// optional ^^^^^
// could also be GMT/UT/EST/A..I/M..Z
const char *ptr;
time_t t;
memset(tm, 0, sizeof(struct tm));
ptr = strptime(date, "%a,%n", tm); // Looks for optional day of week
if (!ptr)
ptr = date;
ptr = strptime(ptr, "%d%n%b%n%Y%n%H:%M:%S%n", tm);
if (!ptr)
{
// date is junk, using current time
time(&t);
gmtime_r(&t, tm);
}
// TODO - adjust the TZ?
}
// Makes a request to Apple based on the Apple Podcast ID in rss_url. The JSON
// response is parsed to find the original feed's url. Example rss_url:
// https://podcasts.apple.com/is/podcast/cgp-grey/id974722423
static char *
apple_rss_feedurl_get(const char *rss_url)
{
struct http_client_ctx ctx;
struct evbuffer *evbuf;
char url[100];
const char *ptr;
unsigned podcast_id;
json_object *jresponse;
json_object *jfeedurl;
char *feedurl;
int ret;
ptr = strrchr(rss_url, '/');
if (!ptr)
{
DPRINTF(E_LOG, L_LIB, "Could not parse Apple Podcast RSS ID from '%s'\n", rss_url);
return NULL;
}
ret = sscanf(ptr, "/id%u", &podcast_id);
if (ret != 1)
{
DPRINTF(E_LOG, L_LIB, "Could not parse Apple Podcast RSS ID from '%s'\n", rss_url);
return NULL;
}
CHECK_NULL(L_LIB, evbuf = evbuffer_new());
snprintf(url, sizeof(url), "%slookup?id=%u", APPLE_ITUNES_SERVER, podcast_id);
memset(&ctx, 0, sizeof(struct http_client_ctx));
ctx.url = url;
ctx.input_body = evbuf;
ret = http_client_request(&ctx);
if (ret < 0 || ctx.response_code != HTTP_OK)
{
evbuffer_free(evbuf);
return NULL;
}
jresponse = jparse_obj_from_evbuffer(evbuf);
evbuffer_free(evbuf);
if (!jresponse)
{
DPRINTF(E_LOG, L_LIB, "Could not parse RSS Apple response, podcast id %u\n", podcast_id);
return NULL;
}
/* expect json resp - get feedUrl
* {
* "resultCount": 1,
* "results": [
* {
* "wrapperType": "track",
* "kind": "podcast",
* ...
* "collectionViewUrl": "https://podcasts.apple.com/us/podcast/cgp-grey/id974722423?uo=4",
* "feedUrl": "http://cgpgrey.libsyn.com/rss",
* ...
* "genres": [
* "Education",
* "Podcasts",
* "News"
* ]
* }
* ]
*}
*/
jfeedurl = JPARSE_SELECT(jresponse, "results", "feedUrl");
if (!jfeedurl || json_object_get_type(jfeedurl) != json_type_string)
{
DPRINTF(E_LOG, L_LIB, "Could not find RSS feedUrl in response from Apple, podcast id %u\n", podcast_id);
jparse_free(jresponse);
return NULL;
}
feedurl = safe_strdup(json_object_get_string(jfeedurl));
DPRINTF(E_DBG, L_LIB, "Mapped Apple podcast URL: '%s' -> '%s'\n", rss_url, feedurl);
jparse_free(jresponse);
return feedurl;
}
static struct playlist_info *
playlist_fetch(bool *is_new, const char *path)
{
struct playlist_info *pli;
int ret;
pli = db_pl_fetch_bypath(path);
if (pli)
{
db_pl_clear_items(pli->id);
*is_new = false;
return pli;
}
CHECK_NULL(L_SCAN, pli = calloc(1, sizeof(struct playlist_info)));
ret = playlist_fill(pli, path);
if (ret < 0)
goto error;
pli->directory_id = DIR_HTTP;
pli->type = PL_RSS;
pli->query_limit = RSS_LIMIT_DEFAULT;
ret = library_playlist_save(pli);
if (ret < 0)
goto error;
pli->id = ret;
*is_new = true;
return pli;
error:
DPRINTF(E_LOG, L_SCAN, "Error adding playlist for RSS feed '%s'\n", path);
free_pli(pli, 0);
return NULL;
}
static mxml_node_t *
rss_xml_get(const char *url)
{
struct http_client_ctx ctx = { 0 };
const char *raw = NULL;
mxml_node_t *xml = NULL;
char *feedurl;
int ret;
// Is it an apple podcast stream?
// ie https://podcasts.apple.com/is/podcast/cgp-grey/id974722423
if (strncmp(url, APPLE_PODCASTS_SERVER, strlen(APPLE_PODCASTS_SERVER)) == 0)
{
feedurl = apple_rss_feedurl_get(url);
if (!feedurl)
return NULL;
}
else
feedurl = strdup(url);
CHECK_NULL(L_LIB, ctx.input_body = evbuffer_new());
ctx.url = feedurl;
ret = http_client_request(&ctx);
if (ret < 0 || ctx.response_code != HTTP_OK)
{
DPRINTF(E_LOG, L_LIB, "Failed to fetch RSS from '%s' (return %d, error code %d)\n", ctx.url, ret, ctx.response_code);
goto cleanup;
}
evbuffer_add(ctx.input_body, "", 1);
raw = (const char*)evbuffer_pullup(ctx.input_body, -1);
xml = mxmlLoadString(NULL, raw, MXML_OPAQUE_CALLBACK);
if (!xml)
{
DPRINTF(E_LOG, L_LIB, "Failed to parse RSS XML from '%s'\n", ctx.url);
goto cleanup;
}
cleanup:
evbuffer_free(ctx.input_body);
free(feedurl);
return xml;
}
static int
rss_xml_parse_feed(const char **feed_title, const char **feed_author, mxml_node_t *xml)
{
mxml_node_t *channel;
mxml_node_t *node;
channel = mxmlFindElement(xml, xml, "channel", NULL, NULL, MXML_DESCEND);
if (!channel)
{
DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml, missing 'channel' node\n");
return -1;
}
node = mxmlFindElement(channel, channel, "title", NULL, NULL, MXML_DESCEND_FIRST);
if (!node)
{
DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml, missing 'title' node\n");
return -1;
}
*feed_title = mxmlGetOpaque(node);
node = mxmlFindElement(channel, channel, "itunes:author", NULL, NULL, MXML_DESCEND_FIRST);
*feed_author = node ? mxmlGetOpaque(node) : NULL;
return 0;
}
static int
rss_xml_parse_item(struct rss_item_info *ri, mxml_node_t *xml, void **saveptr)
{
mxml_node_t *item;
mxml_node_t *node;
const char *s;
if (*saveptr)
{
item = (mxml_node_t *)(*saveptr);
while ( (item = mxmlGetNextSibling(item)) )
{
s = mxmlGetElement(item);
if (s && strcmp(s, "item") == 0)
break;
}
*saveptr = item;
}
else
{
item = mxmlFindElement(xml, xml, "item", NULL, NULL, MXML_DESCEND);
*saveptr = item;
}
if (!item)
return -1; // No more items
memset(ri, 0, sizeof(struct rss_item_info));
node = mxmlFindElement(item, item, "title", NULL, NULL, MXML_DESCEND_FIRST);
ri->title = mxmlGetOpaque(node);
node = mxmlFindElement(item, item, "pubDate", NULL, NULL, MXML_DESCEND_FIRST);
ri->pubdate = mxmlGetOpaque(node);
node = mxmlFindElement(item, item, "link", NULL, NULL, MXML_DESCEND_FIRST);
ri->link = mxmlGetOpaque(node);
node = mxmlFindElement(item, item, "enclosure", NULL, NULL, MXML_DESCEND_FIRST);
ri->url = mxmlElementGetAttr(node, "url");
ri->type = mxmlElementGetAttr(node, "type");
return 0;
}
static void
mfi_metadata_fixup(struct media_file_info *mfi, struct rss_item_info *ri, const char *feed_title, const char *feed_author, uint32_t time_added)
{
struct tm tm;
// Always take the meta from media file if possible; some podcasts (Apple) can
// use mp4 streams which tend not to have decent tags so in those cases take
// info from the RSS and not the stream
if (!mfi->artist)
mfi->artist = safe_strdup(feed_author);
if (!mfi->album)
mfi->album = safe_strdup(feed_title);
if (!mfi->url)
mfi->url = safe_strdup(ri->link);
if (!mfi->genre || strcmp("(186)Podcast", mfi->genre) == 0)
{
free(mfi->genre);
mfi->genre = strdup("Podcast");
}
// The title from the xml is usually better quality
if (ri->title)
{
free(mfi->title);
mfi->title = strdup(ri->title);
}
// Remove, some can be very verbose
free(mfi->comment);
mfi->comment = NULL;
// Date is always from the RSS feed info
rss_date(&tm, ri->pubdate);
mfi->date_released = mktime(&tm);
mfi->year = 1900 + tm.tm_year;
mfi->media_kind = MEDIA_KIND_PODCAST;
mfi->time_added = time_added;
}
static int
rss_save(struct playlist_info *pli, int *count, enum rss_scan_type scan_type)
{
mxml_node_t *xml;
const char *feed_title;
const char *feed_author;
struct media_file_info mfi = { 0 };
struct rss_item_info ri;
uint32_t time_added;
void *ptr = NULL;
int ret;
xml = rss_xml_get(pli->path);
if (!xml)
{
DPRINTF(E_LOG, L_LIB, "Could not get RSS/xml from '%s' (id %d)\n", pli->path, pli->id);
return -1;
}
ret = rss_xml_parse_feed(&feed_title, &feed_author, xml);
if (ret < 0)
{
DPRINTF(E_LOG, L_LIB, "Invalid RSS/xml received from '%s' (id %d)\n", pli->path, pli->id);
mxmlDelete(xml);
return -1;
}
free(pli->title);
pli->title = safe_strdup(feed_title);
// Fake the time - useful when we are adding a new stream - since the
// newest podcasts are added first (the stream is most recent first)
// having time_added date which is older on the most recent episodes
// makes no sense so make all the dates the same for a singleu update
time_added = (uint32_t)time(NULL);
// Walk through the xml, saving each item
*count = 0;
db_transaction_begin();
while ((ret = rss_xml_parse_item(&ri, xml, &ptr)) == 0 && (*count < pli->query_limit))
{
if (library_is_exiting())
{
db_transaction_rollback();
mxmlDelete(xml);
return -1;
}
if (!ri.url)
{
DPRINTF(E_WARN, L_LIB, "Missing URL for item '%s' (date %s) in RSS feed '%s'\n", ri.title, ri.pubdate, feed_title);
continue;
}
db_pl_add_item_bypath(pli->id, ri.url);
(*count)++;
// Try to just ping if already in library
if (scan_type == RSS_SCAN_RESCAN)
{
ret = db_file_ping_bypath(ri.url, 0);
if (ret > 0)
continue;
}
scan_metadata_stream(&mfi, ri.url);
mfi_metadata_fixup(&mfi, &ri, feed_title, feed_author, time_added);
library_media_save(&mfi);
free_mfi(&mfi, 1);
}
db_transaction_end();
mxmlDelete(xml);
return 0;
}
static int
rss_scan(const char *path, enum rss_scan_type scan_type)
{
struct playlist_info *pli;
bool pl_is_new;
int count;
int ret;
// Fetches or creates playlist, clears playlistitems
pli = playlist_fetch(&pl_is_new, path);
if (!pli)
return -1;
// Retrieves the RSS and reads the feed, saving each item as a track, and also
// adds the relationship to playlistitems. The pli will also be updated with
// metadata from the RSS.
ret = rss_save(pli, &count, scan_type);
if (ret < 0)
goto error;
// Save the playlist again, title etc may have been modified by rss_save().
// This also updates the db_timestamp which protects the RSS from deletion.
ret = library_playlist_save(pli);
if (ret < 0)
goto error;
DPRINTF(E_INFO, L_SCAN, "Added or updated %d items from RSS feed '%s' (id %d)\n", count, path, pli->id);
free_pli(pli, 0);
return 0;
error:
if (pl_is_new)
db_pl_delete(pli->id);
free_pli(pli, 0);
return -1;
}
static void
rss_scan_all(enum rss_scan_type scan_type)
{
struct query_params qp = { 0 };
struct db_playlist_info dbpli;
time_t start;
time_t end;
int count;
int ret;
DPRINTF(E_DBG, L_LIB, "Refreshing RSS feeds\n");
start = time(NULL);
qp.type = Q_PL;
qp.sort = S_PLAYLIST;
qp.filter = db_mprintf("(f.type = %d)", PL_RSS);
ret = db_query_start(&qp);
if (ret < 0)
{
DPRINTF(E_LOG, L_LIB, "Failed to find current RSS feeds from db\n");
free(qp.filter);
return;
}
count = 0;
while (((ret = db_query_fetch_pl(&qp, &dbpli)) == 0) && (dbpli.path))
{
ret = rss_scan(dbpli.path, scan_type);
if (ret == 0)
count++;
}
db_query_end(&qp);
free(qp.filter);
end = time(NULL);
if (count == 0)
return;
library_callback_schedule(rss_refresh, NULL, &rss_refresh_interval, LIBRARY_CB_ADD_OR_REPLACE);
DPRINTF(E_INFO, L_LIB, "Refreshed %d RSS feeds in %.f sec (scan type %d)\n", count, difftime(end, start), scan_type);
}
static void
rss_refresh(void *arg)
{
rss_scan_all(RSS_SCAN_RESCAN);
}
static int
rss_rescan(void)
{
rss_scan_all(RSS_SCAN_RESCAN);
return LIBRARY_OK;
}
static int
rss_metascan(void)
{
rss_scan_all(RSS_SCAN_META);
return LIBRARY_OK;
}
static int
rss_fullscan(void)
{
DPRINTF(E_LOG, L_LIB, "RSS feeds removed during full-rescan\n");
return LIBRARY_OK;
}
static int
rss_add(const char *path)
{
int ret;
if (strncmp(path, "http://", 7) != 0 && strncmp(path, "https://", 8) != 0)
{
DPRINTF(E_SPAM, L_LIB, "Invalid RSS path '%s'\n", path);
return LIBRARY_PATH_INVALID;
}
DPRINTF(E_DBG, L_LIB, "Adding RSS '%s'\n", path);
ret = rss_scan(path, RSS_SCAN_RESCAN);
if (ret < 0)
return LIBRARY_PATH_INVALID;
library_callback_schedule(rss_refresh, NULL, &rss_refresh_interval, LIBRARY_CB_ADD_OR_REPLACE);
return LIBRARY_OK;
}
struct library_source rssscanner =
{
.name = "RSS feeds",
.disabled = 0,
.initscan = rss_rescan,
.rescan = rss_rescan,
.metarescan = rss_metascan,
.fullrescan = rss_fullscan,
.item_add = rss_add,
};

90
src/mxml-compat.h
View File

@ -1,6 +1,96 @@
#ifndef __MXML_COMPAT_H__
#define __MXML_COMPAT_H__
// mxml 2.10 has a memory leak in mxmlDelete, see https://github.com/michaelrsweet/mxml/issues/183
// - and since this is the version in Ubuntu 18.04 LTS and Raspian Stretch, we
// fix it by including a fixed mxmlDelete here. It should be removed once the
// major distros no longer have 2.10. The below code is msweet's fixed mxml.
#ifndef HAVE_MXML_OLD
# include <mxml.h>
#else
// Trick to undefine mxml.h's mxmlDelete
#define mxmlDelete mxmlDelete_memleak
# include <mxml.h>
#undef mxmlDelete
static void
compat_mxml_free(mxml_node_t *node)
{
int i;
switch (node->type)
{
case MXML_ELEMENT :
if (node->value.element.name)
free(node->value.element.name);
if (node->value.element.num_attrs)
{
for (i = 0; i < node->value.element.num_attrs; i ++)
{
if (node->value.element.attrs[i].name)
free(node->value.element.attrs[i].name);
if (node->value.element.attrs[i].value)
free(node->value.element.attrs[i].value);
}
free(node->value.element.attrs);
}
break;
case MXML_INTEGER :
break;
case MXML_OPAQUE :
if (node->value.opaque)
free(node->value.opaque);
break;
case MXML_REAL :
break;
case MXML_TEXT :
if (node->value.text.string)
free(node->value.text.string);
break;
case MXML_CUSTOM :
if (node->value.custom.data &&
node->value.custom.destroy)
(*(node->value.custom.destroy))(node->value.custom.data);
break;
default :
break;
}
free(node);
}
static void
mxmlDelete(mxml_node_t *node)
{
mxml_node_t *current,
*next;
if (!node)
return;
mxmlRemove(node);
for (current = node->child; current; current = next)
{
if ((next = current->child) != NULL)
{
current->child = NULL;
continue;
}
if ((next = current->next) == NULL)
{
if ((next = current->parent) == node)
next = NULL;
}
compat_mxml_free(current);
}
compat_mxml_free(node);
}
#endif
/* For compability with mxml 2.6 */
#ifndef HAVE_MXMLGETTEXT
__attribute__((unused)) static const char * /* O - Text string or NULL */