Big performance wins

contrib/sqlite.sql

@@ -1,6 +1,12 @@
+/*
+ * $Id$
+ * 
+ * SQLite database schema
+ */
+
 CREATE TABLE songs (
 	id		INTEGER PRIMARY KEY NOT NULL,
-	path		VARCHAR(4096) NOT NULL,
+	path		VARCHAR(4096) UNIQUE NOT NULL,
 	fname		VARCHAR(255) NOT NULL,
         title		VARCHAR(1024) DEFAULT NULL,
 	artist 		VARCHAR(1024) DEFAULT NULL,
@@ -34,12 +40,13 @@ CREATE TABLE songs (
 	time_played	INTEGER	DEFAULT 0,
 	db_timestamp	INTEGER DEFAULT 0,
 	disabled        INTEGER DEFAULT 0,
-	updated		INTEGER DEFAULT 0,
+	sample_count    INTEGER DEFAULT 0,
 	force_update	INTEGER DEFAULT 0
 );	
 
 CREATE TABLE config (
 	term		VARCHAR(255)	NOT NULL,
+	subterm		VARCHAR(255)    DEFAULT NULL,   
 	value		VARCHAR(1024)	NOT NULL
 );
 
@@ -62,10 +69,10 @@ CREATE TABLE users (
        name	       VARCHAR(255) NOT NULL,
        password	       VARCHAR(255) NOT NULL
 );
-       
-;CREATE INDEX idx_path on songs(path);
 
-INSERT INTO config VALUES ('version','1');
+CREATE INDEX idx_path on songs(path);
+
+INSERT INTO config VALUES ('version','','1');
 INSERT INTO users VALUES (1,1,'admin','mt-daapd');
 INSERT INTO playlists VALUES (1,'Library',1,0,'1');
 

src/db-generic.c

@@ -52,11 +52,11 @@ typedef struct tag_db_functions {
     int(*dbs_enum_fetch)(DBQUERYINFO *, unsigned char **);
     int(*dbs_enum_reset)(DBQUERYINFO *);
     int(*dbs_enum_end)(void);
-    int(*dbs_get_id)(char *);
     int(*dbs_start_scan)(void);
     int(*dbs_end_scan)(void);
     int(*dbs_get_count)(CountType_t);
     MP3FILE*(*dbs_fetch_item)(int);
+    MP3FILE*(*dbs_fetch_path)(char *);
     void(*dbs_dispose_item)(MP3FILE*);
 }DB_FUNCTIONS;
 
@@ -75,11 +75,11 @@ DB_FUNCTIONS db_functions[] = {
 	db_sqlite_enum_fetch,
 	db_sqlite_enum_reset,
 	db_sqlite_enum_end,
-	db_sqlite_get_id,
 	db_sqlite_start_scan,
 	db_sqlite_end_scan,
 	db_sqlite_get_count,
 	db_sqlite_fetch_item,
+	db_sqlite_fetch_path,
 	db_sqlite_dispose_item
     },
 #endif
@@ -423,19 +423,12 @@ int db_scanning(void) {
  */
 int db_add(MP3FILE *pmp3) {
     int retval;
-    int id;
-
-    id=db_get_id(pmp3->path);
 
     db_writelock();
-    if(id) {
+    retval=db_current->dbs_add(pmp3);
-	retval=db_current->dbs_update(pmp3);
+    db_revision_no++;
-    } else {
-	retval=db_current->dbs_add(pmp3);
-	db_revision_no++;
-    }
-
     db_unlock();
+
     return retval;
 }
 
@@ -517,6 +510,16 @@ MP3FILE *db_fetch_item(int id) {
     return retval;
 }
 
+MP3FILE *db_fetch_path(char *path) {
+    MP3FILE *retval;
+    
+    db_readlock();
+    retval=db_current->dbs_fetch_path(path);
+    db_unlock();
+
+    return retval;
+}
+
 int db_start_scan(void) {
     int retval;
 
@@ -538,33 +541,7 @@ int db_end_scan(void) {
     
     return retval;
 }
-
-int db_last_modified(char *path) {
-    int id;
-    MP3FILE *pmp3;
-    int retval=0;
-
-    id=db_get_id(path);
-    pmp3=db_fetch_item(id);
-    if(pmp3) {
-	retval=pmp3->db_timestamp;
-	db_dispose_item(pmp3);
-    }
-    return retval;
-}
     
-
-    
-int db_get_id(char *path) {
-    int retval;
-
-    db_readlock();
-    retval=db_current->dbs_get_id(path);
-    db_unlock();
-
-    return retval;
-}
-
 void db_dispose_item(MP3FILE *pmp3) {
     return db_current->dbs_dispose_item(pmp3);
 }

src/db-generic.h

@@ -138,8 +138,12 @@ extern int db_start_scan(void);
 extern int db_end_scan(void);
 extern int db_exists(char *path);
 extern int db_scanning(void);
-extern int db_last_modified(char *path);
 
+extern MP3FILE *db_fetch_item(int id);
+extern MP3FILE *db_fetch_path(char *path);
+
+
+/* metatag parsing */
 extern MetaField_t db_encode_meta(char *meta);
 extern int db_wantsmeta(MetaField_t meta, MetaFieldName_t fieldNo);
 
@@ -155,8 +159,6 @@ extern int db_dmap_add_container(char *where, char *tag, int size);
  */
 extern int db_get_song_count(void); 
 extern int db_get_playlist_count(void);
-extern MP3FILE *db_fetch_item(int id);
-extern int db_get_id(char *path);
 extern void db_dispose_item(MP3FILE *pmp3);
 
 #endif /* _DB_GENERIC_H_ */

src/dbs-sqlite.c

@@ -43,6 +43,7 @@ static sqlite *db_sqlite_songs; /**< Database that holds the mp3 info */
 static pthread_mutex_t db_sqlite_mutex = PTHREAD_MUTEX_INITIALIZER; /**< sqlite not reentrant */
 static sqlite_vm *db_sqlite_pvm;
 static int db_sqlite_in_scan=0;
+static int db_sqlite_reload=0;
 
 /* Forwards */
 int db_sqlite_get_size(DBQUERYINFO *pinfo, char **valarray);
@@ -97,9 +98,14 @@ int db_sqlite_open(char *parameters) {
 int db_sqlite_init(int reload) {
     int err;
     char *perr;
+    int items;
 
-    if(reload) {
+    items=db_sqlite_get_count(countSongs);
+
+    if((reload) || (!items)) {
+	DPRINTF(E_LOG,L_DB,"Full reload...\n");
 	/* this may or may not fail, depending if the index is already in place */
+	db_sqlite_reload=1;
 	db_sqlite_lock();
 	sqlite_exec(db_sqlite_songs,"DROP INDEX idx_path",NULL,NULL,&perr);
 	err=sqlite_exec(db_sqlite_songs,"DELETE FROM songs",NULL,NULL,&perr);
@@ -129,15 +135,20 @@ int db_sqlite_deinit(void) {
  */
 int db_sqlite_start_scan(void) {
     char *perr;
-    int err;
 
     db_sqlite_lock();
-    err=sqlite_exec(db_sqlite_songs,"UPDATE songs SET updated=0",NULL,NULL,&perr);
+
+    if(db_sqlite_reload) {
+	sqlite_exec(db_sqlite_songs,"PRAGMA synchronous = OFF;",NULL,NULL,&perr);
+	sqlite_exec(db_sqlite_songs,"BEGIN TRANSACTION;",NULL,NULL,&perr);
+    } else {
+	/* if not a full reload, we'll be doing update checks */
+	sqlite_exec(db_sqlite_songs,"DROP TABLE updated",NULL,NULL,&perr);
+	sqlite_exec(db_sqlite_songs,"CREATE TEMP TABLE updated (id int)",NULL,NULL,&perr);
+    }
+
     db_sqlite_unlock();
 
-    if(err != SQLITE_OK)
-	DPRINTF(E_FATAL,L_DB,"db_sqlite_start_scan: %s\n",perr);
-    
     db_sqlite_in_scan=1;
 
     return 0;
@@ -148,16 +159,32 @@ int db_sqlite_start_scan(void) {
  */
 int db_sqlite_end_scan(void) {
     char *perr;
-    int err;
+    int err=0;
 
     db_sqlite_lock();
-    sqlite_exec(db_sqlite_songs,"PRAGMA synchronous=NORMAL",NULL,NULL,&perr);
+    if(db_sqlite_reload) {
-    err=sqlite_exec(db_sqlite_songs,"DELETE FROM songs WHERE updated=0",NULL,NULL,&perr);
+	err=sqlite_exec(db_sqlite_songs,"COMMIT TRANSACTION;",NULL,NULL,&perr);
+	if(err != SQLITE_OK) {
+	    DPRINTF(E_FATAL,L_DB,"Could not commit insert transactions\n");
+	}
+	DPRINTF(E_LOG,L_DB,"Creating path index...\n");
+	sqlite_exec(db_sqlite_songs,"CREATE INDEX idx_path ON songs(path)",NULL,NULL,&perr);
+	sqlite_exec(db_sqlite_songs,"PRAGMA synchronous=NORMAL;",NULL,NULL,&perr);
+    } else {
+	err=sqlite_exec(db_sqlite_songs,
+			"DELETE FROM songs WHERE id NOT IN (SELECT id FROM updated)",
+			NULL,NULL,&perr);
+	if(err == SQLITE_OK) {
+	    err |= sqlite_exec(db_sqlite_songs,"DROP TABLE updated",NULL,NULL,&perr);
+	}
+    }
+
     db_sqlite_unlock();
 
     if(err != SQLITE_OK)
 	DPRINTF(E_FATAL,L_DB,"db_sqlite_end_scan: %s\n",perr);
-    
+
+    db_sqlite_reload=0;
     db_sqlite_in_scan=0;
     return 0;
 }
@@ -174,6 +201,7 @@ int db_sqlite_add(MP3FILE *pmp3) {
 
     DPRINTF(E_SPAM,L_DB,"Entering db_sqlite_add\n");
 
+
     if(!pmp3->time_added)
 	pmp3->time_added = (int)time(NULL);
     
@@ -221,7 +249,7 @@ int db_sqlite_add(MP3FILE *pmp3) {
 			   "%d,"    // time_played
 			   "%d,"    // db_timestamp
 			   "%d,"    // disabled
-			   "1,"     // updated
+			   "%d,"    // sample_count
 			   "0)",    // force_update    
 			   NULL,NULL,
 			   &perr,
@@ -259,10 +287,27 @@ int db_sqlite_add(MP3FILE *pmp3) {
 			   pmp3->db_timestamp,
 			   pmp3->disabled);
     db_sqlite_unlock();
+
+    if(err == SQLITE_CONSTRAINT) {
+	/* probably because the path already exists... */
+	DPRINTF(E_DBG,L_DB,"Could not add mp3 file: %s... updating instead\n",pmp3->path);
+	return db_sqlite_update(pmp3);
+    }
+
     if(err != SQLITE_OK)
 	DPRINTF(E_FATAL,L_DB,"Error inserting file %s in database: %s\n",
 		pmp3->fname,perr);
 
+    if((db_sqlite_in_scan)&&(!db_sqlite_reload)) {
+	db_sqlite_lock();
+	err=sqlite_exec(db_sqlite_songs,"INSERT INTO updated VALUES (last_insert_rowid())",
+			NULL,NULL,&perr);	
+	if(err != SQLITE_OK)
+	    DPRINTF(E_LOG,L_DB,"Error inserting into update table: %s\n",perr);
+	
+	db_sqlite_unlock();
+    }
+
     DPRINTF(E_SPAM,L_DB,"Exiting db_sqlite_add\n");
     return 0;
 }
@@ -308,7 +353,7 @@ int db_sqlite_update(MP3FILE *pmp3) {
 			   "bpm=%d,"    // bpm
 			   "compilation=%d,"    // compilation
 			   "rating=%d,"    // rating
-			   "updated=1"     // updated
+			   "sample_count=%d,"
 			   " WHERE path='%q'",
 			   NULL,NULL,
 			   &perr,
@@ -337,12 +382,24 @@ int db_sqlite_update(MP3FILE *pmp3) {
 			   pmp3->bpm,
 			   pmp3->compilation,
 			   pmp3->rating,
+			   pmp3->sample_count,
 			   pmp3->path);
     db_sqlite_unlock();
     if(err != SQLITE_OK)
 	DPRINTF(E_FATAL,L_DB,"Error updating file %s in database: %s\n",
 		pmp3->fname,perr);
 
+    if((db_sqlite_in_scan) && (!db_sqlite_reload)) {
+	db_sqlite_lock();
+	err=sqlite_exec_printf(db_sqlite_songs,
+			       "INSERT INTO updated (id) select id from songs where path='%q')",
+			       NULL,NULL,&perr,pmp3->path);	
+	if(err != SQLITE_OK)
+	    DPRINTF(E_LOG,L_DB,"Error inserting into update table: %s\n",perr);
+	
+	db_sqlite_unlock();
+    }
+
     return 0;
 }
 
@@ -876,33 +933,6 @@ int db_sqlite_build_dmap(DBQUERYINFO *pinfo, char **valarray, char *presult, int
     return 0;
 }
 
-/**
- * return the id for a given path (or 0 if it does not exist)
- *
- * \param path path of the file we are looking for
- */
-int db_sqlite_get_id(char *path) {
-    int err, rows, cols;
-    char *perr;
-    char **resarray;
-    int retval=0;
-    
-    db_sqlite_lock();
-    err=sqlite_get_table_printf(db_sqlite_songs,"SELECT * FROM songs WHERE path='%q'",
-				&resarray, &rows, &cols, &perr, path);
-    db_sqlite_unlock();
-
-    if(rows != 0) {
-	retval=atoi(resarray[cols]);
-    }
-
-    db_sqlite_lock();
-    sqlite_free_table(resarray);
-    db_sqlite_unlock();
-
-    return retval;
-}
-
 int db_sqlite_atoi(const char *what) {
     return what ? atoi(what) : 0;
 }
@@ -948,6 +978,8 @@ void db_sqlite_build_mp3file(char **valarray, MP3FILE *pmp3) {
     pmp3->time_played=db_sqlite_atoi(valarray[32]);
     pmp3->db_timestamp=db_sqlite_atoi(valarray[33]);
     pmp3->disabled=db_sqlite_atoi(valarray[34]);
+    pmp3->sample_count=db_sqlite_atoi(valarray[35]);
+    pmp3->force_update=db_sqlite_atoi(valarray[36]);
 }
 
 /**
@@ -976,8 +1008,8 @@ MP3FILE *db_sqlite_fetch_item(int id) {
 
     db_sqlite_lock();
     sqlite_free_table(resarray);
-    if(db_sqlite_in_scan) {
+    if((db_sqlite_in_scan) && (!db_sqlite_reload)) {
-	sqlite_exec_printf(db_sqlite_songs,"UPDATE songs SET updated=1 WHERE id=%d",
+	sqlite_exec_printf(db_sqlite_songs,"insert into updated values (%d)",
 			   NULL,NULL,&perr,id);
     }
     db_sqlite_unlock();
@@ -985,6 +1017,45 @@ MP3FILE *db_sqlite_fetch_item(int id) {
     return pmp3;
 }
 
+/**
+ * retrieve a MP3FILE struct for the song with a give path
+ *
+ * \param path path of the file to retreive
+ */
+extern MP3FILE *db_sqlite_fetch_path(char *path) {
+    int err,rows,cols;
+    char *perr;
+    char **resarray;
+    MP3FILE *pmp3=NULL;
+
+    /* here's a bit of a hack */
+    if(db_sqlite_reload) {
+	return NULL;
+    }
+
+    db_sqlite_lock();
+    err=sqlite_get_table_printf(db_sqlite_songs,"SELECT * FROM songs WHERE path=%q",
+				&resarray, &rows, &cols, &perr, path);
+    db_sqlite_unlock();
+
+    if(rows != 0) {
+	pmp3=(MP3FILE*)malloc(sizeof(MP3FILE));
+	if(!pmp3) 
+	    DPRINTF(E_FATAL,L_MISC,"Malloc error in db_sqlite_fetch_item\n");
+
+	db_sqlite_build_mp3file((char **)&resarray[cols],pmp3);
+    }
+
+    db_sqlite_lock();
+    sqlite_free_table(resarray);
+    if((db_sqlite_in_scan)&&(!db_sqlite_reload)) {
+	sqlite_exec_printf(db_sqlite_songs,"INSERT INTO updated VALUES (%d)",
+			   NULL,NULL,&perr,pmp3->id);
+    }
+    db_sqlite_unlock();
+
+    return pmp3;
+}
 
 /** 
  * dispose of a MP3FILE struct that was obtained 
@@ -993,7 +1064,7 @@ MP3FILE *db_sqlite_fetch_item(int id) {
  * \param pmp3 item obtained from db_sqlite_fetch_item
  */
 void db_sqlite_dispose_item(MP3FILE *pmp3) {
-    if(pmp3)
+    if(!pmp3)
 	return;
 
     MAYBEFREE(pmp3->path);

src/dbs-sqlite.h

@@ -32,11 +32,11 @@ extern int db_sqlite_enum_size(DBQUERYINFO *pinfo, int *count);
 extern int db_sqlite_enum_fetch(DBQUERYINFO *pinfo, unsigned char **pdmap);
 extern int db_sqlite_enum_reset(DBQUERYINFO *pinfo);
 extern int db_sqlite_enum_end(void);
-extern int db_sqlite_get_id(char *path);
 extern int db_sqlite_start_scan(void);
 extern int db_sqlite_end_scan(void);
 extern int db_sqlite_get_count(CountType_t type);
 extern MP3FILE *db_sqlite_fetch_item(int id);
+extern MP3FILE *db_sqlite_fetch_path(char *path);
 extern void db_sqlite_dispose_item(MP3FILE *pmp3);
 
 #endif /* _DBS_SQLITE_H_ */

src/mp3-scanner.c

@@ -383,6 +383,7 @@ int scan_path(char *path) {
     struct stat sb;
     int modified_time;
     char *ext;
+    MP3FILE *pmp3;
 
     if((current_dir=opendir(path)) == NULL) {
 	DPRINTF(E_WARN,L_SCAN,"opendir: %s\n",strerror(errno));
@@ -427,18 +428,19 @@ int scan_path(char *path) {
 		    if((strcasecmp(".m3u",(char*)&pde->d_name[strlen(pde->d_name) - 4]) == 0) &&
 		       config.process_m3u){
 			/* we found an m3u file */
-			DPRINTF(E_LOG,L_SCAN,"Oops... we aren't doing playlists.. Sorry: %s\n",
+			DPRINTF(E_LOG,L_SCAN,"Oops... no playlists.. Sorry: %s\n",
 				mp3_path);
 			//			scan_static_playlist(path, pde, &sb);
 		    } else if (((ext = strrchr(pde->d_name, '.')) != NULL) &&
 			       (strcasestr(config.extensions, ext))) {
 			/* only scan if it's been changed, or empty db */
 			modified_time=sb.st_mtime;
-			DPRINTF(E_DBG,L_SCAN,"FS Mod time: %d\n",modified_time);
+			pmp3=db_fetch_path(mp3_path);
-			DPRINTF(E_DBG,L_SCAN,"DB Mod time: %d\n",db_last_modified(mp3_path));
+
-			if(!db_get_id(mp3_path) ||
+			if((!pmp3) || (pmp3->db_timestamp < modified_time) || 
-			   db_last_modified(mp3_path) < modified_time) {
+			   (pmp3->force_update)) {
 			    scan_music_file(path,pde,&sb);
+			    db_dispose_item(pmp3);
 			} else {
 			    DPRINTF(E_DBG,L_SCAN,"Skipping file... not modified\n");
 			}

src/mp3-scanner.h

@@ -66,7 +66,8 @@ typedef struct tag_mp3file {
     char* description;		/* long file type */
     int item_kind;		/* song or movie */
     int data_kind;              /* dmap.datakind (asdk) */
-
+    int force_update;
+    int sample_count;
     char compilation;
 } MP3FILE;