Merge pull request #173 from pi-hole/release/v2.13

Pi-hole FTL v2.13

FTL.h

@@ -66,17 +66,13 @@
 // Default -60 (one minute before a full hour)
 #define GCdelay (-60)
 
-// How often do we dump into FTL's database?
-// Default: 60 (once per minute)
-#define DBinterval 60
-
 // Static structs
 typedef struct {
 	const char* conf;
 	const char* log;
 	const char* pid;
 	const char* port;
-	const char* db;
+	char* db;
 } FTLFileNamesStruct;
 
 typedef struct {
@@ -125,6 +121,7 @@ typedef struct {
 	int maxDBdays;
 	bool resolveIPv6;
 	bool resolveIPv4;
+	int DBinterval;
 } ConfigStruct;
 
 // Dynamic structs

README.md

@@ -111,6 +111,8 @@ Possible settings (**the option shown first is the default**):
 - `MAXDBDAYS=365` (How long should queries be stored in the database? Setting this to `0` disables the database altogether)
 - `RESOLVE_IPV6=yes|no` (Should `FTL` try to resolve IPv6 addresses to host names?)
 - `RESOLVE_IPV4=yes|no` (Should `FTL` try to resolve IPv4 addresses to host names?)
+- `DBINTERVAL=1.0` (How often do we store queries in FTL's database [minutes]?)
+- `DBFILE=/etc/pihole/pihole-FTL.db` (Specify path and filename of FTL's SQLite long-term database. Setting this to `DBFILE=` disables the database altogether)
 
 ### Implemented keywords (starting with `>`, subject to change):
 
@@ -185,7 +187,7 @@ unique_clients 3
 2 10.0 5.6.7.8 some.other.dns.com
 ```
  Variant: `>forward-dest unsorted` to show forward destinations in unsorted order (equivalent to using `>forward-names`)
-```
+
 
 - `>querytypes` : get collected query types percentage
  ```

args.c

@@ -70,7 +70,10 @@ void parse_args(int argc, char* argv[])
 		if(strcmp(argv[i], "-v") == 0 ||
 		   strcmp(argv[i], "version") == 0)
 		{
-			if(strcmp(GIT_BRANCH, "master") == 0)
+			char version[] = GIT_VERSION;
+			// Check if version is of format vX.YY
+			// '.' can never be part of a commit hash
+			if(strstr(version, ".") != NULL)
 				printf("%s\n",GIT_VERSION);
 			else
 				printf("vDev-%s\n",GIT_HASH);
@@ -113,7 +116,8 @@ void parse_args(int argc, char* argv[])
 		{
 			travis = true;
 			FTLfiles.log = "pihole-FTL.log";
-			FTLfiles.db = "pihole-FTL.db";
+			// FTLfiles.db will be set to "pihole-FTL.db" via config file on Travis
+			FTLfiles.conf = "pihole-FTL.conf";
 			files.log = "pihole.log";
 			ok = true;
 		}

config.c

@@ -28,7 +28,7 @@ void read_FTLconf(void)
 	}
 
 	// Parse lines in the config file
-	logg("Starting config file parsing");
+	logg("Starting config file parsing (%s)", FTLfiles.conf);
 
 	// SOCKET_LISTENING
 	// defaults to: listen only local
@@ -130,8 +130,53 @@ void read_FTLconf(void)
 	else
 		logg("   RESOLVE_IPV4: Don\'t resolve IPv4 addresses");
 
+	// DBINTERVAL
+	// How often do we store queries in FTL's database [minutes]?
+	// this value can be a floating point number, e.g. "DBINTERVAL=0.5"
+	// defaults to: once per minute
+	config.DBinterval = 60;
+	buffer = parse_FTLconf(fp, "DBINTERVAL");
+
+	float fvalue = 0;
+	if(buffer != NULL && sscanf(buffer, "%f", &fvalue))
+		// check if the read value is
+		// - larger than 0.1min (6sec), and
+		// - smaller than 1440.0min (once a day)
+		if(fvalue >= 0.1 && fvalue <= 1440.0)
+			config.DBinterval = (int)(60.*fvalue);
+
+	if(config.DBinterval == 60)
+		logg("   DBINTERVAL: saving to DB file every minute");
+	else
+		logg("   DBINTERVAL: saving to DB file every %i seconds", config.DBinterval);
+
+	// DBFILE
+	// defaults to: "/etc/pihole/pihole-FTL.db"
+	buffer = parse_FTLconf(fp, "DBFILE");
+
+	errno = 0;
+	// Use sscanf() to obtain filename from config file parameter only if buffer != NULL
+	if(!(buffer != NULL && sscanf(buffer, "%127ms", &FTLfiles.db)))
+	{
+		// Use standard path if no custom path was obtained from the config file
+		FTLfiles.db = strdup("/etc/pihole/pihole-FTL.db");
+	}
+
+	// Test if memory allocation was successful
+	if(FTLfiles.db == NULL && errno != 0)
+	{
+		logg("FATAL: Allocating memory for FTLfiles.db failed (%s, %i). Exiting.", strerror(errno), errno);
+		exit(EXIT_FAILURE);
+	}
+	else if(FTLfiles.db != NULL && strlen(FTLfiles.db) > 0)
+		logg("   DBFILE: Using %s", FTLfiles.db);
+	else
+		logg("   DBFILE: Not using database due to empty filename");
+
+
 	logg("Finished config file parsing");
 
+	// Release memory
 	if(conflinebuffer != NULL)
 	{
 		free(conflinebuffer);

database.c

@@ -19,6 +19,21 @@ pthread_mutex_t dblock;
 
 enum { DB_VERSION, DB_LASTTIMESTAMP };
 
+void check_database(int rc)
+{
+	// We will retry if the database is busy at the moment
+	// However, we won't retry if any other error happened
+	// and - instead - disable the database functionality
+	// altogether in FTL (setting database to false)
+	if(rc != SQLITE_OK &&
+	   rc != SQLITE_DONE &&
+	   rc != SQLITE_ROW &&
+	   rc != SQLITE_BUSY)
+	{
+		database = false;
+	}
+}
+
 void dbclose(void)
 {
 	sqlite3_close(db);
@@ -51,6 +66,7 @@ bool dbopen(void)
 	if( rc ){
 		logg("dbopen() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return false;
 	}
 
@@ -81,6 +97,7 @@ bool dbquery(const char *format, ...)
 	if( rc != SQLITE_OK ){
 		logg("dbquery(%s) - SQL error (%i): %s", query, rc, zErrMsg);
 		sqlite3_free(zErrMsg);
+		check_database(rc);
 		return false;
 	}
 
@@ -97,6 +114,7 @@ bool db_create(void)
 	if( rc ){
 		logg("db_create() - SQL error (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return false;
 	}
 	// Create Queries table in the database
@@ -128,10 +146,19 @@ bool db_create(void)
 
 void db_init(void)
 {
+	// First check if the user doesn't want to use the database and set an
+	// empty string as file name in FTL's config file
+	if(FTLfiles.db == NULL || strlen(FTLfiles.db) == 0)
+	{
+		database = false;
+		return;
+	}
+
 	int rc = sqlite3_open_v2(FTLfiles.db, &db, SQLITE_OPEN_READWRITE, NULL);
 	if( rc ){
 		logg("db_init() - Cannot open database (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 
 		logg("Creating new (empty) database");
 		if (!db_create())
@@ -172,6 +199,7 @@ int db_get_FTL_property(unsigned int ID)
 	if( rc ){
 		logg("db_get_FTL_property() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return -1;
 	}
 	free(querystring);
@@ -181,6 +209,7 @@ int db_get_FTL_property(unsigned int ID)
 	if( rc != SQLITE_ROW ){
 		logg("db_get_FTL_property() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return -1;
 	}
 
@@ -205,13 +234,15 @@ int number_of_queries_in_DB(void)
 	if( rc ){
 		logg("number_of_queries_in_DB() - SQL error prepare (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return -1;
 	}
 
 	rc = sqlite3_step(stmt);
 	if( rc != SQLITE_ROW ){
 		logg("number_of_queries_in_DB() - SQL error step (%i): %s", rc, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return -1;
 	}
 
@@ -274,6 +305,7 @@ void save_to_DB(void)
 	{
 		logg("save_to_DB() - error in preparing SQL statement (%i): %s", ret, sqlite3_errmsg(db));
 		dbclose();
+		check_database(rc);
 		return;
 	}
 
@@ -338,6 +370,8 @@ void save_to_DB(void)
 				logg("save_to_DB() - exiting due to too many errors");
 				break;
 			}
+			// Check this error message
+			check_database(rc);
 		}
 
 		saved++;
@@ -411,18 +445,17 @@ void *DB_thread(void *val)
 	// Set thread name
 	prctl(PR_SET_NAME,"DB",0,0,0);
 
-	if(!DBdeleteoldqueries)
-	{
-		// Lock FTL's data structure, since it is likely that it will be changed here
-		enable_thread_lock("DB_thread");
+	// Lock FTL's data structure, since it is likely that it will be changed here
+	enable_thread_lock("DB_thread");
 
-		// Save data to database
-		save_to_DB();
+	// Save data to database
+	save_to_DB();
 
-		// Release thread lock
-		disable_thread_lock("DB_thread");
-	}
-	else
+	// Release thread lock
+	disable_thread_lock("DB_thread");
+
+	// Check if GC should be done on the database
+	if(DBdeleteoldqueries)
 	{
 		// No thread locks needed
 		delete_old_queries_in_DB();

log.c

@@ -144,7 +144,8 @@ void log_counter_info(void)
 void log_FTL_version(void)
 {
 	logg("FTL branch: %s", GIT_BRANCH);
-	logg("FTL hash: %s", GIT_VERSION);
+	logg("FTL version: %s", GIT_VERSION);
+	logg("FTL tag: %s", GIT_TAG);
 	logg("FTL date: %s", GIT_DATE);
 	logg("FTL user: %s", username);
 }

main.c

@@ -86,7 +86,7 @@ int main (int argc, char* argv[]) {
 		if(((time(NULL) - GCdelay)%GCinterval) == 0)
 			runGCthread = true;
 
-		if(database && ((time(NULL)%DBinterval) == 0))
+		if(database && ((time(NULL)%config.DBinterval) == 0))
 			runDBthread = true;
 
 		// Garbadge collect in regular interval, but don't do it if the threadlocks is set
@@ -134,7 +134,7 @@ int main (int argc, char* argv[]) {
 			}
 
 			// Avoid immediate re-run of DB thread
-			while(((time(NULL)%DBinterval) == 0))
+			while(((time(NULL)%config.DBinterval) == 0))
 				sleepms(100);
 		}
 

parser.c

@@ -336,9 +336,10 @@ void process_pihole_log(int file)
 			char *domainwithspaces = calloc(domainlen+3,sizeof(char));
 			// strncat() NULL-terminates the copied string (strncpy() doesn't!)
 			strncat(domain,domainstart+2,domainlen);
+			// Copy string into buffer surrounded by spaces
+			sprintf(domainwithspaces," %s ",domain);
 			// Convert domain to lower case
 			strtolower(domain);
-			sprintf(domainwithspaces," %s ",domain);
 
 			if(strcmp(domain, "pi.hole") == 0)
 			{

request.c

@@ -1038,7 +1038,8 @@ void getVersion(int *sock)
 {
 	char server_message[SOCKETBUFFERLEN];
 
-	if(strcmp(GIT_BRANCH, "master") == 0)
+	char version[] = GIT_VERSION;
+	if(strstr(version, ".") != NULL)
 		sprintf(server_message,"version %s\ntag %s\nbranch %s\ndate %s\n", GIT_VERSION, GIT_TAG, GIT_BRANCH, GIT_DATE);
 	else
 		sprintf(server_message,"version vDev-%s\ntag %s\nbranch %s\ndate %s\n", GIT_HASH, GIT_TAG, GIT_BRANCH, GIT_DATE);

structs.c

@@ -15,7 +15,7 @@ FTLFileNamesStruct FTLfiles = {
 	"/var/log/pihole-FTL.log",
 	"/var/run/pihole-FTL.pid",
 	"/var/run/pihole-FTL.port",
-	"/etc/pihole/pihole-FTL.db"
+	NULL
 };
 
 logFileNamesStruct files = {

test/run.sh

@@ -10,12 +10,12 @@ ts="$(dnsmasq_pre)"
 cat <<EOT >> pihole.log
 ${ts} query[AAAA] raspberrypi from 127.0.0.1
 ${ts} /etc/pihole/local.list raspberrypi is fda2:2001:5647:0:ba27:ebff:fe37:4205
-${ts} query[A] checkip.dyndns.org from 127.0.0.1
-${ts} forwarded checkip.dyndns.org to 2001:1608:10:25::9249:d69b
-${ts} forwarded checkip.dyndns.org to 2001:1608:10:25::1c04:b12f
-${ts} forwarded checkip.dyndns.org to 2620:0:ccd::2
-${ts} forwarded checkip.dyndns.org to 2620:0:ccc::2
-${ts} reply checkip.dyndns.org is <CNAME>
+${ts} query[A] ChEcKiP.DyNdNs.OrG from 127.0.0.1
+${ts} forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::9249:d69b
+${ts} forwarded ChEcKiP.DyNdNs.OrG to 2001:1608:10:25::1c04:b12f
+${ts} forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccd::2
+${ts} forwarded ChEcKiP.DyNdNs.OrG to 2620:0:ccc::2
+${ts} reply ChEcKiP.DyNdNs.OrG is <CNAME>
 ${ts} reply checkip.dyndns.com is 216.146.38.70
 ${ts} reply checkip.dyndns.com is 216.146.43.71
 ${ts} reply checkip.dyndns.com is 91.198.22.70
@@ -44,6 +44,10 @@ ${ts} /etc/pihole/gravity.list addomain.com is 1.2.3.4
 EOT
 touch "pihole-FTL.log"
 
+cat <<EOT >> pihole-FTL.conf
+DBFILE=pihole-FTL.db
+EOT
+
 # Start FTL
 ./pihole-FTL travis-ci