mirror of
https://github.com/owntone/owntone-server.git
synced 2024-12-25 06:35:57 -05:00
Add slow DAAP queries to cache automatically
This commit is contained in:
ejurgensen
parent
cf091e8d8b
commit
31ef4d4e13
@ -11,8 +11,11 @@ general {
|
||||
admin_password = "unused"
|
||||
# Enable/disable IPv6
|
||||
ipv6 = no
|
||||
# Location of DAAP cache (comment the line to disable caching)
|
||||
daapcache_path = "/var/cache/forked-daapd/daapcache.db"
|
||||
# Location of DAAP cache
|
||||
# daapcache_path = "/var/cache/forked-daapd/daapcache.db"
|
||||
# DAAP requests that take longer than this threshold (in msec) get their
|
||||
# replies cached for next time. Set to 0 to disable caching.
|
||||
# daapcache_threshold = 1000
|
||||
}
|
||||
|
||||
# Library configuration
|
||||
|
||||
@ -53,7 +53,8 @@ static cfg_opt_t sec_general[] =
|
||||
CFG_INT("db_pragma_synchronous", -1, CFGF_NONE),
|
||||
CFG_INT_CB("loglevel", E_LOG, CFGF_NONE, &cb_loglevel),
|
||||
CFG_BOOL("ipv6", cfg_false, CFGF_NONE),
|
||||
CFG_STR("daapcache_path", NULL, CFGF_NONE),
|
||||
CFG_STR("daapcache_path", STATEDIR "/cache/" PACKAGE "/daapcache.db", CFGF_NONE),
|
||||
CFG_INT("daapcache_threshold", 1000, CFGF_NONE),
|
||||
CFG_END()
|
||||
};
|
||||
|
||||
|
||||
232
src/daap_cache.c
232
src/daap_cache.c
@ -37,44 +37,12 @@
|
||||
#include "db.h"
|
||||
#include "daap_cache.h"
|
||||
|
||||
/* The DAAP cache will only cache raw daap replies for these queries.
|
||||
* Remove session_id and revision-number from the query, if you add a new one.
|
||||
/* The DAAP cache will cache raw daap replies for queries added with
|
||||
* daapcache_add(). Only some query types are supported.
|
||||
* You can't add queries where the canonical reply is not HTTP_OK, because
|
||||
* daap_request will use that as default for cache replies.
|
||||
*
|
||||
* TODO: Don't hardcode, detect slow queries and add them dynamically
|
||||
*/
|
||||
struct daapcache_query_t
|
||||
{
|
||||
const char *ua;
|
||||
const char *query;
|
||||
};
|
||||
|
||||
static const struct daapcache_query_t daapcache_queries[] =
|
||||
{
|
||||
// Remote 4.2, Playlist 1 - Library
|
||||
{ "Remote", "/databases/1/containers/1/items?meta=dmap.itemname,dmap.itemid,daap.songartist,daap.songalbumartist,daap.songalbum,com.apple.itunes.cloud-id,dmap.containeritemid,com.apple.itunes.has-video,com.apple.itunes.itms-songid,com.apple.itunes.extended-media-kind,dmap.downloadstatus,daap.songdisabled&type=music&sort=name&include-sort-headers=1&query=('com.apple.itunes.extended-media-kind:1','com.apple.itunes.extended-media-kind:32')" },
|
||||
// Remote 4.2, Albums
|
||||
{ "Remote", "/databases/1/groups?meta=dmap.itemname,dmap.itemid,dmap.persistentid,daap.songartist,com.apple.itunes.cloud-id,daap.songartistid,daap.songalbumid,dmap.persistentid,daap.songtime,daap.songdatereleased,dmap.downloadstatus&type=music&group-type=albums&sort=album&include-sort-headers=0&query=('daap.songalbum!:'%2B('com.apple.itunes.extended-media-kind:1','com.apple.itunes.extended-media-kind:32'))" },
|
||||
// Remote 4.2, Artists
|
||||
{ "Remote", "/databases/1/groups?meta=dmap.itemname,dmap.itemid,dmap.persistentid,daap.songartist,daap.groupalbumcount,daap.songartistid&type=music&group-type=artists&sort=album&include-sort-headers=1&query=('daap.songartist!:'%2B('com.apple.itunes.extended-media-kind:1','com.apple.itunes.extended-media-kind:32'))" },
|
||||
// iTunes 11.3, DB song list
|
||||
{ "iTunes", "/databases/1/items?delta=0&type=music&meta=all" },
|
||||
// iTunes 11.3, Playlist 1 - Library
|
||||
{ "iTunes", "/databases/1/containers/1/items?delta=0&type=music&meta=dmap.itemkind,dmap.itemid,dmap.containeritemid" },
|
||||
// iTunes 11.3, Playlist 2 - Music
|
||||
{ "iTunes", "/databases/1/containers/2/items?delta=0&type=music&meta=dmap.itemkind,dmap.itemid,dmap.containeritemid" },
|
||||
// TunesRemote+, Albums
|
||||
{ "Remote", "/databases/1/groups?meta=dmap.itemname,dmap.itemid,dmap.persistentid,daap.songartist&type=music&group-type=albums&sort=album&include-sort-headers=1" },
|
||||
// TunesRemote+, Artists
|
||||
{ "Remote", "/databases/1/browse/artists?include-sort-headers=1" },
|
||||
// Retune, Artists
|
||||
{ "Remote", "/databases/1/groups?meta=dmap.itemname,dmap.itemid,dmap.persistentid,daap.songartist,daap.groupalbumcount&type=music&group-type=artists&sort=album&include-sort-headers=1&query=(('com.apple.itunes.mediakind:1','com.apple.itunes.mediakind:32')+'daap.songartist!:')" },
|
||||
// Retune, Albums
|
||||
{ "Remote", "/databases/1/groups?meta=dmap.itemname,dmap.itemid,dmap.persistentid,daap.songartist,daap.songdatereleased,dmap.itemcount,daap.songtime&type=music&group-type=albums&sort=album&include-sort-headers=1&query=(('com.apple.itunes.mediakind:1','com.apple.itunes.mediakind:32')+'daap.songalbum!:')" },
|
||||
// Retune, Playlist 1 - Library
|
||||
{ "Remote", "/databases/1/containers/1/items?meta=dmap.itemname,dmap.itemid,daap.songartist,daap.songalbum,daap.songtime,dmap.containeritemid,com.apple.tunes.has-video,com.apple.itunes.can-be-genius-seed&type=music&sort=artist&include-sort-headers=1&query=(('com.apple.itunes.mediakind:1','com.apple.itunes.mediakind:32'))" },
|
||||
};
|
||||
|
||||
struct daapcache_command;
|
||||
|
||||
@ -90,7 +58,8 @@ struct daapcache_command
|
||||
int nonblock;
|
||||
|
||||
struct {
|
||||
const char *query;
|
||||
char *query;
|
||||
char *ua;
|
||||
struct evbuffer *evbuf;
|
||||
} arg;
|
||||
|
||||
@ -118,6 +87,10 @@ static char *g_db_path;
|
||||
// After being triggered wait 5 seconds before rebuilding daapcache
|
||||
static struct timeval g_wait = { 5, 0 };
|
||||
|
||||
// The user may configure a threshold (in msec), and queries slower than
|
||||
// that will have their reply cached
|
||||
static int g_cfg_threshold;
|
||||
|
||||
/* --------------------------------- HELPERS ------------------------------- */
|
||||
|
||||
/* The purpose of this function is to remove transient tags from a request
|
||||
@ -231,14 +204,21 @@ thread_exit(void)
|
||||
static int
|
||||
daapcache_create(void)
|
||||
{
|
||||
#define T_CACHE \
|
||||
"CREATE TABLE IF NOT EXISTS cache (" \
|
||||
" id INTEGER PRIMARY KEY NOT NULL," \
|
||||
" query VARCHAR(4096) NOT NULL," \
|
||||
" reply BLOB" \
|
||||
#define T_REPLIES \
|
||||
"CREATE TABLE IF NOT EXISTS replies (" \
|
||||
" id INTEGER PRIMARY KEY NOT NULL," \
|
||||
" query VARCHAR(4096) NOT NULL," \
|
||||
" reply BLOB" \
|
||||
");"
|
||||
#define I_QUERY \
|
||||
"CREATE INDEX IF NOT EXISTS idx_query ON cache(query);"
|
||||
#define T_QUERIES \
|
||||
"CREATE TABLE IF NOT EXISTS queries (" \
|
||||
" id INTEGER PRIMARY KEY NOT NULL," \
|
||||
" query VARCHAR(4096) UNIQUE NOT NULL," \
|
||||
" user_agent VARCHAR(1024)," \
|
||||
" timestamp INTEGER DEFAULT 0" \
|
||||
");"
|
||||
#define I_QUERY \
|
||||
"CREATE INDEX IF NOT EXISTS idx_query ON replies (query);"
|
||||
char *errmsg;
|
||||
int ret;
|
||||
|
||||
@ -255,11 +235,22 @@ daapcache_create(void)
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Create cache table
|
||||
ret = sqlite3_exec(g_db_hdl, T_CACHE, NULL, NULL, &errmsg);
|
||||
// Create reply cache table
|
||||
ret = sqlite3_exec(g_db_hdl, T_REPLIES, NULL, NULL, &errmsg);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
DPRINTF(E_FATAL, L_DCACHE, "Error creating cache table: %s\n", errmsg);
|
||||
DPRINTF(E_FATAL, L_DCACHE, "Error creating reply cache table: %s\n", errmsg);
|
||||
|
||||
sqlite3_free(errmsg);
|
||||
sqlite3_close(g_db_hdl);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Create query table (the queries for which we will generate and cache replies)
|
||||
ret = sqlite3_exec(g_db_hdl, T_QUERIES, NULL, NULL, &errmsg);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
DPRINTF(E_FATAL, L_DCACHE, "Error creating query table: %s\n", errmsg);
|
||||
|
||||
sqlite3_free(errmsg);
|
||||
sqlite3_close(g_db_hdl);
|
||||
@ -303,11 +294,11 @@ daapcache_destroy(void)
|
||||
DPRINTF(E_DBG, L_DCACHE, "Cache destroyed\n");
|
||||
}
|
||||
|
||||
/* Adds the reply in evbuf to the cache */
|
||||
/* Adds the reply (stored in evbuf) to the cache */
|
||||
static int
|
||||
daapcache_query_add(const char *query, struct evbuffer *evbuf)
|
||||
daapcache_reply_add(const char *query, struct evbuffer *evbuf)
|
||||
{
|
||||
#define Q_TMPL "INSERT INTO cache(query, reply) VALUES(?, ?);"
|
||||
#define Q_TMPL "INSERT INTO replies (query, reply) VALUES (?, ?);"
|
||||
sqlite3_stmt *stmt;
|
||||
unsigned char *data;
|
||||
size_t datlen;
|
||||
@ -347,11 +338,80 @@ daapcache_query_add(const char *query, struct evbuffer *evbuf)
|
||||
#undef Q_TMPL
|
||||
}
|
||||
|
||||
/* Adds the query to the list of queries for which we will build and cache a reply */
|
||||
static int
|
||||
daapcache_query_add(struct daapcache_command *cmd)
|
||||
{
|
||||
#define Q_TMPL "INSERT OR REPLACE INTO queries (user_agent, query, timestamp) VALUES ('%q', '%q', %" PRIi64 ");"
|
||||
#define Q_CLEANUP "DELETE FROM queries WHERE id NOT IN (SELECT id FROM queries ORDER BY timestamp DESC LIMIT 20);"
|
||||
char *query;
|
||||
char *errmsg;
|
||||
int ret;
|
||||
|
||||
if (!cmd->arg.ua)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Couldn't add slow query to cache, unknown user-agent\n");
|
||||
|
||||
free(cmd->arg.query);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Currently we are only able to pre-build and cache these reply types
|
||||
if ( (strncmp(cmd->arg.query, "/databases/1/containers/", strlen("/databases/1/containers/")) != 0) &&
|
||||
(strncmp(cmd->arg.query, "/databases/1/groups?", strlen("/databases/1/groups?")) != 0) &&
|
||||
(strncmp(cmd->arg.query, "/databases/1/items?", strlen("/databases/1/items?")) != 0) &&
|
||||
(strncmp(cmd->arg.query, "/databases/1/browse/", strlen("/databases/1/browse/")) != 0) )
|
||||
return -1;
|
||||
|
||||
remove_tag(cmd->arg.query, "session-id");
|
||||
remove_tag(cmd->arg.query, "revision-number");
|
||||
|
||||
query = sqlite3_mprintf(Q_TMPL, cmd->arg.ua, cmd->arg.query, (int64_t)time(NULL));
|
||||
if (!query)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Out of memory making query string.\n");
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
ret = sqlite3_exec(g_db_hdl, query, NULL, NULL, &errmsg);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error adding query to query list: %s\n", errmsg);
|
||||
|
||||
sqlite3_free(query);
|
||||
sqlite3_free(errmsg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
sqlite3_free(query);
|
||||
|
||||
DPRINTF(E_DBG, L_DCACHE, "Added query to query list (user-agent %s): %s\n", cmd->arg.ua, cmd->arg.query);
|
||||
|
||||
free(cmd->arg.ua);
|
||||
free(cmd->arg.query);
|
||||
|
||||
// Limits the size of the cache to only contain replies for 20 most recent queries
|
||||
ret = sqlite3_exec(g_db_hdl, Q_CLEANUP, NULL, NULL, &errmsg);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error cleaning up query list before update: %s\n", errmsg);
|
||||
sqlite3_free(errmsg);
|
||||
return -1;
|
||||
}
|
||||
|
||||
daapcache_trigger();
|
||||
|
||||
return 0;
|
||||
#undef Q_CLEANUP
|
||||
#undef Q_TMPL
|
||||
}
|
||||
|
||||
/* Gets a reply from the cache */
|
||||
static int
|
||||
daapcache_query_get(struct daapcache_command *cmd)
|
||||
{
|
||||
#define Q_TMPL "SELECT reply FROM cache WHERE query = ?;"
|
||||
#define Q_TMPL "SELECT reply FROM replies WHERE query = ?;"
|
||||
sqlite3_stmt *stmt;
|
||||
char *query;
|
||||
int datlen;
|
||||
@ -359,7 +419,7 @@ daapcache_query_get(struct daapcache_command *cmd)
|
||||
|
||||
cmd->arg.evbuf = NULL;
|
||||
|
||||
query = strdup(cmd->arg.query);
|
||||
query = cmd->arg.query;
|
||||
remove_tag(query, "session-id");
|
||||
remove_tag(query, "revision-number");
|
||||
|
||||
@ -423,27 +483,34 @@ daapcache_query_get(struct daapcache_command *cmd)
|
||||
static void
|
||||
daapcache_update_cb(int fd, short what, void *arg)
|
||||
{
|
||||
sqlite3_stmt *stmt;
|
||||
struct evbuffer *evbuf;
|
||||
char *errmsg;
|
||||
char *query;
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
DPRINTF(E_INFO, L_DCACHE, "Timeout reached, time to update DAAP cache\n");
|
||||
|
||||
ret = sqlite3_exec(g_db_hdl, "DELETE FROM cache;", NULL, NULL, &errmsg);
|
||||
ret = sqlite3_exec(g_db_hdl, "DELETE FROM replies;", NULL, NULL, &errmsg);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error clearing cache before update: %s\n", errmsg);
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error clearing reply cache before update: %s\n", errmsg);
|
||||
sqlite3_free(errmsg);
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < (sizeof(daapcache_queries) / sizeof(daapcache_queries[0])); i++)
|
||||
ret = sqlite3_prepare_v2(g_db_hdl, "SELECT user_agent, query FROM queries;", -1, &stmt, 0);
|
||||
if (ret != SQLITE_OK)
|
||||
{
|
||||
query = strdup(daapcache_queries[i].query);
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error preparing for cache update: %s\n", sqlite3_errmsg(g_db_hdl));
|
||||
return;
|
||||
}
|
||||
|
||||
evbuf = daap_reply_build(query, daapcache_queries[i].ua);
|
||||
while ((ret = sqlite3_step(stmt)) == SQLITE_ROW)
|
||||
{
|
||||
query = strdup((char *)sqlite3_column_text(stmt, 1));
|
||||
|
||||
evbuf = daap_reply_build(query, (char *)sqlite3_column_text(stmt, 0));
|
||||
if (!evbuf)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Error building DAAP reply for query: %s\n", query);
|
||||
@ -451,12 +518,17 @@ daapcache_update_cb(int fd, short what, void *arg)
|
||||
continue;
|
||||
}
|
||||
|
||||
daapcache_query_add(query, evbuf);
|
||||
daapcache_reply_add(query, evbuf);
|
||||
|
||||
free(query);
|
||||
evbuffer_free(evbuf);
|
||||
}
|
||||
|
||||
if (ret != SQLITE_DONE)
|
||||
DPRINTF(E_LOG, L_DCACHE, "Could not step: %s\n", sqlite3_errmsg(g_db_hdl));
|
||||
|
||||
sqlite3_finalize(stmt);
|
||||
|
||||
DPRINTF(E_INFO, L_DCACHE, "DAAP cache updated\n");
|
||||
}
|
||||
|
||||
@ -604,7 +676,7 @@ daapcache_get(const char *query)
|
||||
command_init(&cmd);
|
||||
|
||||
cmd.func = daapcache_query_get;
|
||||
cmd.arg.query = query;
|
||||
cmd.arg.query = strdup(query);
|
||||
|
||||
ret = sync_command(&cmd);
|
||||
|
||||
@ -615,15 +687,55 @@ daapcache_get(const char *query)
|
||||
return ((ret < 0) ? NULL : evbuf);
|
||||
}
|
||||
|
||||
void
|
||||
daapcache_add(const char *query, const char *ua)
|
||||
{
|
||||
struct daapcache_command *cmd;
|
||||
|
||||
if (!g_initialized)
|
||||
return;
|
||||
|
||||
cmd = (struct daapcache_command *)malloc(sizeof(struct daapcache_command));
|
||||
if (!cmd)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Could not allocate daapcache_command\n");
|
||||
return;
|
||||
}
|
||||
|
||||
memset(cmd, 0, sizeof(struct daapcache_command));
|
||||
|
||||
cmd->nonblock = 1;
|
||||
|
||||
cmd->func = daapcache_query_add;
|
||||
cmd->arg.query = strdup(query);
|
||||
cmd->arg.ua = strdup(ua);
|
||||
|
||||
nonblock_command(cmd);
|
||||
}
|
||||
|
||||
int
|
||||
daapcache_threshold(void)
|
||||
{
|
||||
return g_cfg_threshold;
|
||||
}
|
||||
|
||||
int
|
||||
daapcache_init(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
g_db_path = cfg_getstr(cfg_getsec(cfg, "general"), "daapcache_path");
|
||||
if (!g_db_path)
|
||||
if (!g_db_path || (strlen(g_db_path) == 0))
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Cache disabled\n");
|
||||
DPRINTF(E_LOG, L_DCACHE, "Cache path invalid, disabling cache\n");
|
||||
g_initialized = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
g_cfg_threshold = cfg_getint(cfg_getsec(cfg, "general"), "daapcache_threshold");
|
||||
if (g_cfg_threshold == 0)
|
||||
{
|
||||
DPRINTF(E_LOG, L_DCACHE, "Cache threshold set to 0, disabling cache\n");
|
||||
g_initialized = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -15,6 +15,12 @@ daapcache_trigger(void);
|
||||
struct evbuffer *
|
||||
daapcache_get(const char *query);
|
||||
|
||||
void
|
||||
daapcache_add(const char *query, const char *ua);
|
||||
|
||||
int
|
||||
daapcache_threshold(void);
|
||||
|
||||
int
|
||||
daapcache_init(void);
|
||||
|
||||
|
||||
@ -34,9 +34,11 @@
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <inttypes.h>
|
||||
#include <time.h>
|
||||
#include <ctype.h>
|
||||
|
||||
#include <uninorm.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <avl.h>
|
||||
|
||||
@ -2525,10 +2527,13 @@ daap_request(struct evhttp_request *req)
|
||||
struct evbuffer *evbuf;
|
||||
struct evkeyvalq query;
|
||||
struct evkeyvalq *headers;
|
||||
struct timespec start;
|
||||
struct timespec end;
|
||||
const char *ua;
|
||||
cfg_t *lib;
|
||||
char *libname;
|
||||
char *passwd;
|
||||
int msec;
|
||||
int handler;
|
||||
int ret;
|
||||
int i;
|
||||
@ -2690,8 +2695,19 @@ daap_request(struct evhttp_request *req)
|
||||
|
||||
evhttp_parse_query(full_uri, &query);
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &start);
|
||||
|
||||
daap_handlers[handler].handler(req, evbuf, uri_parts, &query, ua);
|
||||
|
||||
clock_gettime(CLOCK_MONOTONIC, &end);
|
||||
|
||||
msec = (end.tv_sec * 1000 + end.tv_nsec / 1000000) - (start.tv_sec * 1000 + start.tv_nsec / 1000000);
|
||||
|
||||
DPRINTF(E_DBG, L_DB, "DAAP request handled in %d milliseconds\n", msec);
|
||||
|
||||
if (msec > daapcache_threshold())
|
||||
daapcache_add(full_uri, ua);
|
||||
|
||||
evhttp_clear_headers(&query);
|
||||
evbuffer_free(evbuf);
|
||||
free(uri);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user