Merge branch 'album_meta1'

sqlext/sqlext.c

@@ -31,183 +31,6 @@
 #include <sqlite3ext.h>
 SQLITE_EXTENSION_INIT1
 
-
-/*
- * MurmurHash2, 64-bit versions, by Austin Appleby
- *
- * Code released under the public domain, as per
- *    <http://murmurhash.googlepages.com/>
- * as of 2010-01-03.
- */
-
-#if SIZEOF_VOID_P == 8 /* 64bit platforms */
-
-static uint64_t
-murmur_hash64(const void *key, int len, uint32_t seed)
-{
-  const int r = 47;
-  const uint64_t m = 0xc6a4a7935bd1e995;
-
-  const uint64_t *data;
-  const uint64_t *end;
-  const unsigned char *data_tail;
-  uint64_t h;
-  uint64_t k;
-
-  h = seed ^ (len * m);
-  data = (const uint64_t *)key;
-  end = data + (len / 8);
-
-  while (data != end)
-    {
-      k = *data++;
-
-      k *= m;
-      k ^= k >> r;
-      k *= m;
-
-      h ^= k;
-      h *= m;
-    }
-
-  data_tail = (const unsigned char *)data;
-
-  switch (len & 7)
-    {
-      case 7:
-	h ^= (uint64_t)(data_tail[6]) << 48;
-      case 6:
-	h ^= (uint64_t)(data_tail[5]) << 40;
-      case 5:
-	h ^= (uint64_t)(data_tail[4]) << 32;
-      case 4:
-	h ^= (uint64_t)(data_tail[3]) << 24;
-      case 3:
-	h ^= (uint64_t)(data_tail[2]) << 16;
-      case 2:
-	h ^= (uint64_t)(data_tail[1]) << 8;
-      case 1:
-	h ^= (uint64_t)(data_tail[0]);
-	h *= m;
-    }
-
-  h ^= h >> r;
-  h *= m;
-  h ^= h >> r;
-
-  return h;
-}
-
-#elif SIZEOF_VOID_P == 4 /* 32bit platforms */
-
-static uint64_t
-murmur_hash64(const void *key, int len, uint32_t seed)
-{
-  const int r = 24;
-  const uint32_t m = 0x5bd1e995;
-
-  const uint32_t *data;
-  const unsigned char *data_tail;
-  uint32_t k1;
-  uint32_t h1;
-  uint32_t k2;
-  uint32_t h2;
-
-  uint64_t h;
-
-  h1 = seed ^ len;
-  h2 = 0;
-
-  data = (const uint32_t *)key;
-
-  while (len >= 8)
-    {
-      k1 = *data++;
-      k1 *= m; k1 ^= k1 >> r; k1 *= m;
-      h1 *= m; h1 ^= k1;
-
-      k2 = *data++;
-      k2 *= m; k2 ^= k2 >> r; k2 *= m;
-      h2 *= m; h2 ^= k2;
-
-      len -= 8;
-    }
-
-  if (len >= 4)
-    {
-      k1 = *data++;
-      k1 *= m; k1 ^= k1 >> r; k1 *= m;
-      h1 *= m; h1 ^= k1;
-      len -= 4;
-    }
-
-  data_tail = (const unsigned char *)data;
-
-  switch(len)
-    {
-      case 3:
-	h2 ^= (uint32_t)(data_tail[2]) << 16;
-      case 2:
-	h2 ^= (uint32_t)(data_tail[1]) << 8;
-      case 1:
-	h2 ^= (uint32_t)(data_tail[0]);
-	h2 *= m;
-    };
-
-  h1 ^= h2 >> 18; h1 *= m;
-  h2 ^= h1 >> 22; h2 *= m;
-  h1 ^= h2 >> 17; h1 *= m;
-  h2 ^= h1 >> 19; h2 *= m;
-
-  h = h1;
-  h = (h << 32) | h2;
-
-  return h;
-}
-
-#else
-# error Platform not supported
-#endif
-
-static void
-sqlext_daap_songalbumid_xfunc(sqlite3_context *pv, int n, sqlite3_value **ppv)
-{
-  const char *album_artist;
-  const char *album;
-  char *hashbuf;
-  sqlite3_int64 result;
-
-  if (n != 2)
-    {
-      sqlite3_result_error(pv, "daap_songalbumid() requires 2 parameters, album_artist and album", -1);
-      return;
-    }
-
-  if ((sqlite3_value_type(ppv[0]) != SQLITE_TEXT)
-      || (sqlite3_value_type(ppv[1]) != SQLITE_TEXT))
-    {
-      sqlite3_result_error(pv, "daap_songalbumid() requires 2 text parameters", -1);
-      return;
-    }
-
-  album_artist = (const char *)sqlite3_value_text(ppv[0]);
-  album = (const char *)sqlite3_value_text(ppv[1]);
-
-  hashbuf = sqlite3_mprintf("%s==%s", (album_artist) ? album_artist : "", (album) ? album : "");
-  if (!hashbuf)
-    {
-      sqlite3_result_error(pv, "daap_songalbumid() out of memory for hashbuf", -1);
-      return;
-    }
-
-  /* Limit hash length to 63 bits, due to signed type in sqlite */
-  result = murmur_hash64(hashbuf, strlen(hashbuf), 0) >> 1;
-
-  sqlite3_free(hashbuf);
-
-  sqlite3_result_int64(pv, result);
-}
-
 static void
 sqlext_daap_no_zero_xfunc(sqlite3_context *pv, int n, sqlite3_value **ppv)
 {
@@ -278,15 +101,6 @@ sqlite3_extension_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines
   SQLITE_EXTENSION_INIT2(pApi);
   int ret;
 
-  ret = sqlite3_create_function(db, "daap_songalbumid", 2, SQLITE_UTF8, NULL, sqlext_daap_songalbumid_xfunc, NULL, NULL);
-  if (ret != SQLITE_OK)
-    {
-      if (pzErrMsg)
-	*pzErrMsg = sqlite3_mprintf("Could not create daap_songalbumid function: %s\n", sqlite3_errmsg(db));
-
-      return -1;
-    }
-
   ret = sqlite3_create_function(db, "daap_no_zero", 2, SQLITE_UTF8, NULL, sqlext_daap_no_zero_xfunc, NULL, NULL);
   if (ret != SQLITE_OK)
     {

src/db.c

@@ -92,6 +92,8 @@ enum fixup_type {
   DB_FIXUP_COMPOSER_SORT,
   DB_FIXUP_TIME_ADDED,
   DB_FIXUP_TIME_MODIFIED,
+  DB_FIXUP_SONGARTISTID,
+  DB_FIXUP_SONGALBUMID,
 };
 
 struct db_unlock {
@@ -200,8 +202,8 @@ static const struct col_type_map mfi_cols_map[] =
     { "tv_network_name",    mfi_offsetof(tv_network_name),    DB_TYPE_STRING },
     { "tv_episode_sort",    mfi_offsetof(tv_episode_sort),    DB_TYPE_INT },
     { "tv_season_num",      mfi_offsetof(tv_season_num),      DB_TYPE_INT },
-    { "songartistid",       mfi_offsetof(songartistid),       DB_TYPE_INT64,  DB_FIXUP_STANDARD, DB_FLAG_AUTO },
+    { "songartistid",       mfi_offsetof(songartistid),       DB_TYPE_INT64,  DB_FIXUP_SONGARTISTID },
-    { "songalbumid",        mfi_offsetof(songalbumid),        DB_TYPE_INT64,  DB_FIXUP_STANDARD, DB_FLAG_AUTO },
+    { "songalbumid",        mfi_offsetof(songalbumid),        DB_TYPE_INT64,  DB_FIXUP_SONGALBUMID },
     { "title_sort",         mfi_offsetof(title_sort),         DB_TYPE_STRING, DB_FIXUP_TITLE_SORT },
     { "artist_sort",        mfi_offsetof(artist_sort),        DB_TYPE_STRING, DB_FIXUP_ARTIST_SORT },
     { "album_sort",         mfi_offsetof(album_sort),         DB_TYPE_STRING, DB_FIXUP_ALBUM_SORT },
@@ -726,7 +728,7 @@ sort_tag_create(char **sort_tag, const char *src_tag)
 
   if (*sort_tag)
     {
-      DPRINTF(E_DBG, L_LIB, "Existing sort tag will be normalized: %s\n", *sort_tag);
+      DPRINTF(E_DBG, L_DB, "Existing sort tag will be normalized: %s\n", *sort_tag);
       o_ptr = u8_normalize(UNINORM_NFD, (uint8_t *)*sort_tag, strlen(*sort_tag) + 1, NULL, &len);
       free(*sort_tag);
       *sort_tag = (char *)o_ptr;
@@ -841,7 +843,6 @@ fixup_sanitize(char **tag, enum fixup_type fixup, struct fixup_ctx *ctx)
 	    *tag = ret;
 	  }
     }
-
 }
 
 static void
@@ -851,6 +852,16 @@ fixup_defaults(char **tag, enum fixup_type fixup, struct fixup_ctx *ctx)
 
   switch(fixup)
     {
+      case DB_FIXUP_SONGARTISTID:
+	if (ctx->mfi && ctx->mfi->songartistid == 0)
+	  ctx->mfi->songartistid = two_str_hash(ctx->mfi->album_artist, NULL);
+	break;
+
+      case DB_FIXUP_SONGALBUMID:
+	if (ctx->mfi && ctx->mfi->songalbumid == 0)
+	  ctx->mfi->songalbumid = two_str_hash(ctx->mfi->album_artist, ctx->mfi->album);
+	break;
+
       case DB_FIXUP_TITLE:
 	if (*tag)
 	  break;

src/db_init.c

@@ -400,25 +400,15 @@ static const struct db_init_query db_init_index_queries[] =
 
 /* Triggers must be prefixed with trg_ for db_drop_triggers() to id them */
 
-#define TRG_FILES_INSERT_SONGIDS									\
+#define TRG_GROUPS_INSERT										\
-  "CREATE TRIGGER trg_files_insert_songids AFTER INSERT ON files FOR EACH ROW"				\
+  "CREATE TRIGGER trg_groups_insert AFTER INSERT ON files FOR EACH ROW"					\
   " BEGIN"												\
-  "   UPDATE files SET songartistid = daap_songalbumid(LOWER(NEW.album_artist), ''), "			\
+  "   INSERT OR IGNORE INTO groups (type, name, persistentid) VALUES (1, NEW.album, NEW.songalbumid);"	\
-  "     songalbumid = daap_songalbumid(LOWER(NEW.album_artist), LOWER(NEW.album))"			\
+  "   INSERT OR IGNORE INTO groups (type, name, persistentid) VALUES (2, NEW.album_artist, NEW.songartistid);"	\
-  "   WHERE id = NEW.id;"										\
-  " END;"
-
-#define TRG_FILES_UPDATE_SONGIDS									\
-  "CREATE TRIGGER trg_files_update_songids AFTER UPDATE OF album_artist, album ON files FOR EACH ROW"	\
-  " BEGIN"												\
-  "   UPDATE files SET songartistid = daap_songalbumid(LOWER(NEW.album_artist), ''), "			\
-  "     songalbumid = daap_songalbumid(LOWER(NEW.album_artist), LOWER(NEW.album))"			\
-  "   WHERE id = NEW.id;"										\
   " END;"
 
 #define TRG_GROUPS_UPDATE										\
   "CREATE TRIGGER trg_groups_update AFTER UPDATE OF songartistid, songalbumid ON files FOR EACH ROW"	\
-  " WHEN (NEW.songartistid != 0 AND NEW.songalbumid != 0)"						\
   " BEGIN"												\
   "   INSERT OR IGNORE INTO groups (type, name, persistentid) VALUES (1, NEW.album, NEW.songalbumid);"	\
   "   INSERT OR IGNORE INTO groups (type, name, persistentid) VALUES (2, NEW.album_artist, NEW.songartistid);"	\
@@ -426,8 +416,7 @@ static const struct db_init_query db_init_index_queries[] =
 
 static const struct db_init_query db_init_trigger_queries[] =
   {
-    { TRG_FILES_INSERT_SONGIDS,    "create trigger trg_files_insert_songids" },
+    { TRG_GROUPS_INSERT,           "create trigger trg_groups_insert" },
-    { TRG_FILES_UPDATE_SONGIDS,    "create trigger trg_files_update_songids" },
     { TRG_GROUPS_UPDATE,           "create trigger trg_groups_update" },
   };
 

src/db_init.h

@@ -25,8 +25,8 @@
  * version of the database? If yes, then it is a minor upgrade, if no, then it
  * is a major upgrade. In other words minor version upgrades permit downgrading
  * forked-daapd after the database was upgraded. */
-#define SCHEMA_VERSION_MAJOR 20
+#define SCHEMA_VERSION_MAJOR 21
-#define SCHEMA_VERSION_MINOR 01
+#define SCHEMA_VERSION_MINOR 00
 
 int
 db_init_indices(sqlite3 *hdl);

src/db_upgrade.c

@@ -983,6 +983,18 @@ static const struct db_upgrade_query db_upgrade_v2001_queries[] =
     { U_V2001_SCVER_MINOR,    "set schema_version_minor to 01" },
   };
 
+#define U_V2100_SCVER_MAJOR \
+  "UPDATE admin SET value = '21' WHERE key = 'schema_version_major';"
+#define U_V2100_SCVER_MINOR \
+  "UPDATE admin SET value = '00' WHERE key = 'schema_version_minor';"
+
+// This upgrade just changes triggers (will be done automatically by db_drop...)
+static const struct db_upgrade_query db_upgrade_v2100_queries[] =
+  {
+    { U_V2100_SCVER_MAJOR,    "set schema_version_major to 21" },
+    { U_V2100_SCVER_MINOR,    "set schema_version_minor to 00" },
+  };
+
 
 int
 db_upgrade(sqlite3 *hdl, int db_ver)
@@ -1124,6 +1136,13 @@ db_upgrade(sqlite3 *hdl, int db_ver)
       if (ret < 0)
 	return -1;
 
+      /* FALLTHROUGH */
+
+    case 2001:
+      ret = db_generic_upgrade(hdl, db_upgrade_v2100_queries, ARRAY_SIZE(db_upgrade_v2100_queries));
+      if (ret < 0)
+	return -1;
+
       break;
 
     default:

src/library/filescanner_ffmpeg.c

@@ -128,6 +128,18 @@ parse_date(struct media_file_info *mfi, char *date_string)
   return ret;
 }
 
+static int
+parse_albumid(struct media_file_info *mfi, char *id_string)
+{
+  // Already set by a previous tag that we give higher priority
+  if (mfi->songalbumid)
+    return 0;
+
+  // Limit hash length to 63 bits, due to signed type in sqlite
+  mfi->songalbumid = murmur_hash64(id_string, strlen(id_string), 0) >> 1;
+  return 1;
+}
+
 /* Lookup is case-insensitive, first occurrence takes precedence */
 static const struct metadata_map md_map_generic[] =
   {
@@ -152,6 +164,16 @@ static const struct metadata_map md_map_generic[] =
     { "album-sort",   0, mfi_offsetof(album_sort),         NULL },
     { "compilation",  1, mfi_offsetof(compilation),        NULL },
 
+    // These tags are used to determine if files belong to a common compilation
+    // or album, ref. https://picard.musicbrainz.org/docs/tags
+    { "MusicBrainz Album Id",         1, mfi_offsetof(songalbumid), parse_albumid },
+    { "MusicBrainz Release Group Id", 1, mfi_offsetof(songalbumid), parse_albumid },
+    { "MusicBrainz DiscID",           1, mfi_offsetof(songalbumid), parse_albumid },
+    { "CDDB DiscID",                  1, mfi_offsetof(songalbumid), parse_albumid },
+    { "iTunes_CDDB_IDs",              1, mfi_offsetof(songalbumid), parse_albumid },
+    { "CATALOGNUMBER",                1, mfi_offsetof(songalbumid), parse_albumid },
+    { "BARCODE",                      1, mfi_offsetof(songalbumid), parse_albumid },
+
     { NULL,           0, 0,                                NULL }
   };
 

src/misc.c

@@ -747,6 +747,29 @@ djb_hash(const void *data, size_t len)
   return hash;
 }
 
+int64_t
+two_str_hash(const char *a, const char *b)
+{
+  char hashbuf[2048];
+  int64_t hash;
+  int i;
+  int ret;
+
+  ret = snprintf(hashbuf, sizeof(hashbuf), "%s==%s", (a) ? a : "", (b) ? b : "");
+  if (ret < 0 || ret == sizeof(hashbuf))
+    {
+      DPRINTF(E_LOG, L_MISC, "Buffer too large to calculate hash: '%s==%s'\n", a, b);
+      return 999999; // Stand-in hash...
+    }
+
+  for (i = 0; hashbuf[i]; i++)
+    hashbuf[i] = tolower(hashbuf[i]);
+
+  // Limit hash length to 63 bits, due to signed type in sqlite
+  hash = murmur_hash64(hashbuf, strlen(hashbuf), 0) >> 1;
+
+  return hash;
+}
 
 static unsigned char b64_decode_table[256];
 

src/misc.h

@@ -130,6 +130,9 @@ swap_pointers(char **a, char **b);
 uint32_t
 djb_hash(const void *data, size_t len);
 
+int64_t
+two_str_hash(const char *a, const char *b);
+
 char *
 b64_decode(const char *b64);