Merge pull request #1542 from pi-hole/development

Release v5.22

.github/.codespellignore

@@ -0,0 +1,4 @@
+ssudo
+tre
+ede
+nd

.github/workflows/build.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
       -
         name: "Calculate required variables"
         id: variables
@@ -53,7 +53,7 @@ jobs:
 
     needs: smoke-tests
 
-    container: ghcr.io/pi-hole/ftl-build:v1.23-${{ matrix.arch }}
+    container: ghcr.io/pi-hole/ftl-build:v1.26-${{ matrix.arch }}
 
     strategy:
       fail-fast: false
@@ -81,14 +81,16 @@ jobs:
             bin_name: pihole-FTL-armv8-linux-gnueabihf
           - arch: aarch64
             bin_name: pihole-FTL-aarch64-linux-gnu
+          - arch: riscv64
+            bin_name: pihole-FTL-riscv64-linux-gnu
 
     env:
       CI_ARCH: ${{ matrix.arch }}${{ matrix.arch_extra }}
 
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
       -
         name: "Fix ownership of repository"
         run: chown -R root .
@@ -131,7 +133,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
       -
         name: Get Binaries built in previous jobs
         uses: actions/[email protected]

.github/workflows/codespell.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.4.0
     -
       name: Spell-Checking
       uses: codespell-project/actions-codespell@master
       with:
-        ignore_words_file: .codespellignore
-        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex
+        ignore_words_file: .github/.codespellignore
+        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs

.github/workflows/sync-back-to-dev.yml

@@ -11,7 +11,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

CMakeLists.txt

@@ -11,6 +11,6 @@
 cmake_minimum_required(VERSION 2.8.12)
 project(PIHOLE_FTL C)
 
-set(DNSMASQ_VERSION pi-hole-v2.89)
+set(DNSMASQ_VERSION pi-hole-v2.89-9461807)
 
 add_subdirectory(src)

src/database/gravity-db.c

@@ -47,6 +47,7 @@ static sqlite3 *gravity_db = NULL;
 static sqlite3_stmt* table_stmt = NULL;
 static sqlite3_stmt* auditlist_stmt = NULL;
 bool gravityDB_opened = false;
+static bool gravity_abp_format = false;
 
 // Table names corresponding to the enum defined in gravity-db.h
 static const char* tablename[] = { "vw_gravity", "vw_blacklist", "vw_whitelist", "vw_regex_blacklist", "vw_regex_whitelist" , "" };
@@ -95,6 +96,40 @@ void gravityDB_forked(void)
 	gravityDB_open();
 }
 
+static void gravity_check_ABP_format(void)
+{
+	// Check if we have a valid ABP format
+	// We do this by checking the "abp_domains" property in the "info" table
+
+	// Prepare statement
+	sqlite3_stmt *stmt = NULL;
+	int rc = sqlite3_prepare_v2(gravity_db,
+	                            "SELECT value FROM info WHERE property = 'abp_domains';",
+	                            -1, &stmt, NULL);
+
+	if( rc != SQLITE_OK )
+	{
+		logg("gravity_check_ABP_format() - SQL error prepare: %s", sqlite3_errstr(rc));
+		return;
+	}
+
+	// Execute statement
+	rc = sqlite3_step(stmt);
+	if( rc != SQLITE_ROW )
+	{
+		// No result
+		gravity_abp_format = false;
+		sqlite3_finalize(stmt);
+		return;
+	}
+
+	// Get result (SQLite3 stores 1 for TRUE, 0 for FALSE)
+	gravity_abp_format = sqlite3_column_int(stmt, 0) != 0;
+
+	// Finalize statement
+	sqlite3_finalize(stmt);
+}
+
 // Open gravity database
 bool gravityDB_open(void)
 {
@@ -194,6 +229,10 @@ bool gravityDB_open(void)
 		logg("gravityDB_open() - Cannot set busy handler: %s", sqlite3_errstr(rc));
 	}
 
+	// Check (and remember in global variable) if there are any ABP-style
+	// entries in the database
+	gravity_check_ABP_format();
+
 	if(config.debug & DEBUG_DATABASE)
 		logg("gravityDB_open(): Successfully opened gravity.db");
 	return true;
@@ -1290,7 +1329,102 @@ enum db_result in_gravity(const char *domain, clientsData *client)
 	if(stmt == NULL)
 		stmt = gravity_stmt->get(gravity_stmt, client->id);
 
-	return domain_in_list(domain, stmt, "gravity", NULL);
+	// Check if domain is exactly in gravity list
+	const enum db_result exact_match = domain_in_list(domain, stmt, "gravity", NULL);
+	if(config.debug & DEBUG_QUERIES)
+		logg("Checking if \"%s\" is in gravity: %s",
+		     domain, exact_match == FOUND ? "yes" : "no");
+	// Return for anything else than "not found" (e.g. "found" or "list not available")
+	if(exact_match != NOT_FOUND)
+		return exact_match;
+
+	// Return early if we are not supposed to check for ABP-style regex
+	// matches. This needs to be enabled in the config file as it is
+	// computationally expensive and not needed in most cases (HOSTS lists).
+	if(!gravity_abp_format)
+		return NOT_FOUND;
+
+	// Make a copy of the domain we will slowly truncate
+	// while extracting the individual components below
+	char *domainBuf = strdup(domain);
+
+	// Buffer to hold the constructed (sub)domain in ABP format
+	char *abpDomain = calloc(strlen(domain) + 4, sizeof(char));
+	// Prime abp matcher with minimal content
+	strcpy(abpDomain, "||^");
+
+	// Get number of domain parts (equals the number of dots + 1)
+	unsigned int N = 1u;
+	for(const char *p = domain; *p != '\0'; p++)
+		if(*p == '.')
+			N++;
+
+	// Loop over domain parts, building matcher from the TLD
+	// going down into domain and subdomains one by one
+	while(N-- > 0)
+	{
+		// Get domain to the *last* occurrence of '.'
+		char *ptr = strrchr(domainBuf, '.');
+
+		// If there are no '.' left in the domain buffer, we use the
+		// remainder which is the left-most domain component
+		if(ptr == NULL)
+			ptr = domainBuf;
+
+		// Get size of this component...
+		const size_t component_size = strlen(ptr);
+		// ... and use it to create a "gap" of the right size in our ABP
+		// format buffer
+		// Insert the domain component into the gap
+		if(ptr[0] == '.')
+		{
+			// If the component starts with a dot, we need
+			// to skip it when copying it into the ABP buffer
+			// Move excluding initial "||" but including final \0 (strlen-2+1 = strlen-1)
+			memmove(abpDomain+2+component_size-1, abpDomain+2, strlen(abpDomain)-1);
+			// Copy component bytes (excl. trailing null-byte)
+			memcpy(abpDomain+2, ptr+1, component_size-1);
+		}
+		else
+		{
+			// Otherwise, we copy the component as-is
+			memmove(abpDomain+2+component_size, abpDomain+2, strlen(abpDomain)-1);
+			// Copy component bytes (excl. trailing null-byte)
+			memcpy(abpDomain+2, ptr, component_size);
+		}
+		// Check if the constructed ABP-style domain is in the gravity list
+		const enum db_result abp_match = domain_in_list(abpDomain, stmt, "gravity", NULL);
+		if(config.debug & DEBUG_QUERIES)
+			logg("Checking if \"%s\" is in gravity: %s",
+			     abpDomain, abp_match == FOUND ? "yes" : "no");
+		// Return for anything else than "not found" (e.g. "found" or "list not available")
+		if(abp_match != NOT_FOUND)
+		{
+			free(domainBuf);
+			free(abpDomain);
+			return abp_match;
+		}
+		// Truncate the domain buffer to the left of the
+		// last dot, effectively removing the last component
+		const ssize_t truncate_pos = strlen(domainBuf)-component_size;
+		if(truncate_pos < 1)
+			// This was already the last iteration
+			break;
+
+		// Put a null-byte at the truncation position
+		domainBuf[truncate_pos] = '\0';
+
+		// Move the ABP buffer to the right by one byte ...
+		memmove(abpDomain+3, abpDomain+2, strlen(abpDomain));
+		// ... and insert '.' for the next iteration
+		abpDomain[2] = '.';
+	}
+
+	free(domainBuf);
+	free(abpDomain);
+
+	// Domain not found in gravity list
+	return NOT_FOUND;
 }
 
 enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client)