tpws: hostlist/ipset dedup and auto reload

tpws/hostlist.c

@@ -1,7 +1,6 @@
 #include <stdio.h>
 #include "hostlist.h"
 #include "gzip.h"
-#include "params.h"
 #include "helpers.h"
 
 // inplace tolower() and add to pool
@@ -56,7 +55,7 @@ bool AppendHostList(strpool **hostlist, const char *filename)
 		if (r==Z_OK)
 		{
 			DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
-			
+
 			p = zbuf;
 			e = zbuf + zsize;
 			while(p<e)
@@ -79,7 +78,7 @@ bool AppendHostList(strpool **hostlist, const char *filename)
 	else
 	{
 		DLOG_CONDUP("loading plain text list\n");
-		
+
 		while (fgets(s, sizeof(s), F))
 		{
 			p = s;
@@ -97,22 +96,38 @@ bool AppendHostList(strpool **hostlist, const char *filename)
 	return true;
 }
 
-bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list)
+static bool LoadHostList(struct hostlist_file *hfile)
 {
-	struct str_list *file;
+	time_t t = file_mod_time(hfile->filename);
-
+	if (!t)
-	if (*hostlist)
 	{
-		StrPoolDestroy(hostlist);
+		// stat() error
-		*hostlist = NULL;
+		DLOG_ERR("cannot access hostlist file '%s'. in-memory content remains unchanged.\n",hfile->filename);
+		return true;
 	}
-
+	if (t==hfile->mod_time) return true; // up to date
-	LIST_FOREACH(file, file_list, next)
+	StrPoolDestroy(&hfile->hostlist);
+	if (!AppendHostList(&hfile->hostlist, hfile->filename))
 	{
-		if (!AppendHostList(hostlist, file->str)) return false;
+		StrPoolDestroy(&hfile->hostlist);
+		return false;
 	}
+	hfile->mod_time=t;
 	return true;
 }
+static bool LoadHostLists(struct hostlist_files_head *list)
+{
+	bool bres=true;
+	struct hostlist_file *hfile;
+
+	LIST_FOREACH(hfile, list, next)
+	{
+		if (!LoadHostList(hfile))
+			// at least one failed
+			bres=false;
+	}
+	return bres;
+}
 
 bool NonEmptyHostlist(strpool **hostlist)
 {
@@ -120,8 +135,27 @@ bool NonEmptyHostlist(strpool **hostlist)
 	return *hostlist ? true : StrPoolAddStrLen(hostlist, "@&()", 4);
 }
 
+static void MakeAutolistsNonEmpty()
+{
+	struct desync_profile_list *dpl;
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		if (dpl->dp.hostlist_auto)
+			NonEmptyHostlist(&dpl->dp.hostlist_auto->hostlist);
+	}
+}
 
-bool SearchHostList(strpool *hostlist, const char *host)
+bool LoadAllHostLists()
+{
+	if (!LoadHostLists(&params.hostlists))
+		return false;
+	MakeAutolistsNonEmpty();
+	return true;
+}
+
+
+
+static bool SearchHostList(strpool *hostlist, const char *host)
 {
 	if (hostlist)
 	{
@@ -130,7 +164,7 @@ bool SearchHostList(strpool *hostlist, const char *host)
 		while (p)
 		{
 			bInHostList = StrPoolCheckStr(hostlist, p);
-			VPRINT("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
+			VPRINT("hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
 			if (bInHostList) return true;
 			p = strchr(p, '.');
 			if (p) p++;
@@ -139,73 +173,108 @@ bool SearchHostList(strpool *hostlist, const char *host)
 	return false;
 }
 
-// return : true = apply fooling, false = do not apply
+
-static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
+static bool HostlistsReloadCheck(const struct hostlist_collection_head *hostlists)
 {
-	if (excluded) *excluded = false;
+	struct hostlist_item *item;
-	if (hostlist_exclude)
+	LIST_FOREACH(item, hostlists, next)
 	{
-		VPRINT("Checking exclude hostlist\n");
+		if (!LoadHostList(item->hfile))
-		if (SearchHostList(hostlist_exclude, host))
+			return false;
+	}
+	MakeAutolistsNonEmpty();
+	return true;
+}
+bool HostlistsReloadCheckForProfile(const struct desync_profile *dp)
+{
+	return HostlistsReloadCheck(&dp->hl_collection) && HostlistsReloadCheck(&dp->hl_collection_exclude);
+}
+// return : true = apply fooling, false = do not apply
+static bool HostlistCheck_(const struct hostlist_collection_head *hostlists, const struct hostlist_collection_head *hostlists_exclude, const char *host, bool *excluded, bool bSkipReloadCheck)
+{
+	struct hostlist_item *item;
+
+	if (excluded) *excluded = false;
+
+	if (!bSkipReloadCheck)
+		if (!HostlistsReloadCheck(hostlists) || !HostlistsReloadCheck(hostlists_exclude))
+			return false;
+
+	LIST_FOREACH(item, hostlists_exclude, next)
+	{
+		VPRINT("[%s] exclude ", item->hfile->filename);
+		if (SearchHostList(item->hfile->hostlist, host))
 		{
 			if (excluded) *excluded = true;
 			return false;
 		}
 	}
-	if (hostlist)
+	// old behavior compat: all include lists are empty means check passes
+	if (!hostlist_collection_is_empty(hostlists))
 	{
-		VPRINT("Checking include hostlist\n");
+		LIST_FOREACH(item, hostlists, next)
-		return SearchHostList(hostlist, host);
+		{
-	}
+			VPRINT("[%s] include ", item->hfile->filename);
-	return true;
+			if (SearchHostList(item->hfile->hostlist, host))
-}
+				return true;
-
+		}
-static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
-{
-	if (!LoadHostLists(&dp->hostlist, &dp->hostlist_files))
 		return false;
-	if (*dp->hostlist_auto_filename)
-	{
-		dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
-		NonEmptyHostlist(&dp->hostlist);
 	}
 	return true;
 }
 
-bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
+
+// return : true = apply fooling, false = do not apply
+bool HostlistCheck(const struct desync_profile *dp, const char *host, bool *excluded, bool bSkipReloadCheck)
 {
 	VPRINT("* hostlist check for profile %d\n",dp->n);
-	if (*dp->hostlist_auto_filename)
+	return HostlistCheck_(&dp->hl_collection, &dp->hl_collection_exclude, host, excluded, bSkipReloadCheck);
-	{
-		time_t t = file_mod_time(dp->hostlist_auto_filename);
-		if (t!=dp->hostlist_auto_mod_time)
-		{
-			DLOG_CONDUP("Autohostlist '%s' from profile %d was modified. Reloading include hostlists for this profile.\n",dp->hostlist_auto_filename, dp->n);
-			if (!LoadIncludeHostListsForProfile(dp))
-			{
-				// what will we do without hostlist ?? sure, gonna die
-				exit(1);
-			}
-			dp->hostlist_auto_mod_time = t;
-			NonEmptyHostlist(&dp->hostlist);
-		}
-	}
-	return HostlistCheck_(dp->hostlist, dp->hostlist_exclude, host, excluded);
 }
 
-bool LoadIncludeHostLists()
+
+static struct hostlist_file *RegisterHostlist_(struct hostlist_files_head *hostlists, struct hostlist_collection_head *hl_collection, const char *filename)
 {
-	struct desync_profile_list *dpl;
+	struct hostlist_file *hfile;
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	if (!(hfile=hostlist_files_search(hostlists, filename)))
-		if (!LoadIncludeHostListsForProfile(&dpl->dp))
+		if (!(hfile=hostlist_files_add(hostlists, filename)))
-			return false;
+			return NULL;
-	return true;
+	if (!hostlist_collection_search(hl_collection, filename))
+		if (!hostlist_collection_add(hl_collection, hfile))
+			return NULL;
+	return hfile;
 }
-bool LoadExcludeHostLists()
+struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename)
 {
-	struct desync_profile_list *dpl;
+	if (!file_mod_time(filename))
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
-		if (!LoadHostLists(&dpl->dp.hostlist_exclude, &dpl->dp.hostlist_exclude_files))
+		DLOG_ERR("cannot access hostlist file '%s'\n",filename);
-			return false;
+		return NULL;
-	return true;
+	}
+	return RegisterHostlist_(
+		&params.hostlists,
+		bExclude ? &dp->hl_collection_exclude : &dp->hl_collection,
+		filename);
+}
+
+void HostlistsDebug()
+{
+	if (!params.debug) return;
+
+	struct hostlist_file *hfile;
+	struct desync_profile_list *dpl;
+	struct hostlist_item *hl_item;
+
+	LIST_FOREACH(hfile, &params.hostlists, next)
+		VPRINT("hostlist file %s%s\n",hfile->filename,hfile->hostlist ? "" : " (empty)");
+
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		LIST_FOREACH(hl_item, &dpl->dp.hl_collection, next)
+			if (hl_item->hfile!=dpl->dp.hostlist_auto)
+				VPRINT("profile %d include hostlist %s%s\n",dpl->dp.n, hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+		LIST_FOREACH(hl_item, &dpl->dp.hl_collection_exclude, next)
+			VPRINT("profile %d exclude hostlist %s%s\n",dpl->dp.n,hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
+		if (dpl->dp.hostlist_auto)
+			VPRINT("profile %d auto hostlist %s%s\n",dpl->dp.n,dpl->dp.hostlist_auto->filename,dpl->dp.hostlist_auto->hostlist ? "" : " (empty)");
+	}
 }

tpws/hostlist.h

@@ -5,10 +5,10 @@
 #include "params.h"
 
 bool AppendHostList(strpool **hostlist, const char *filename);
-bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
+bool LoadAllHostLists();
-bool LoadIncludeHostLists();
-bool LoadExcludeHostLists();
 bool NonEmptyHostlist(strpool **hostlist);
-bool SearchHostList(strpool *hostlist, const char *host);
 // return : true = apply fooling, false = do not apply
-bool HostlistCheck(struct desync_profile *dp,const char *host, bool *excluded);
+bool HostlistCheck(const struct desync_profile *dp,const char *host, bool *excluded, bool bSkipReloadCheck);
+struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename);
+bool HostlistsReloadCheckForProfile(const struct desync_profile *dp);
+void HostlistsDebug();

tpws/ipset.c

@@ -3,6 +3,7 @@
 #include "gzip.h"
 #include "helpers.h"
 
+
 // inplace tolower() and add to pool
 static bool addpool(ipset *ips, char **s, const char *end, int *ct)
 {
@@ -75,7 +76,7 @@ static bool AppendIpset(ipset *ips, const char *filename)
 		if (r==Z_OK)
 		{
 			DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
-			
+
 			p = zbuf;
 			e = zbuf + zsize;
 			while(p<e)
@@ -98,7 +99,7 @@ static bool AppendIpset(ipset *ips, const char *filename)
 	else
 	{
 		DLOG_CONDUP("loading plain text list\n");
-		
+
 		while (fgets(s, sizeof(s)-1, F))
 		{
 			p = s;
@@ -116,37 +117,45 @@ static bool AppendIpset(ipset *ips, const char *filename)
 	return true;
 }
 
-static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
+static bool LoadIpset(struct ipset_file *hfile)
 {
-	struct str_list *file;
+	time_t t = file_mod_time(hfile->filename);
-
+	if (!t)
-	ipsetDestroy(ips);
-
-	LIST_FOREACH(file, file_list, next)
 	{
-		if (!AppendIpset(ips, file->str)) return false;
+		// stat() error
+		DLOG_ERR("cannot access ipset file '%s'. in-memory content remains unchanged.\n",hfile->filename);
+		return true;
 	}
+	if (t==hfile->mod_time) return true; // up to date
+	ipsetDestroy(&hfile->ipset);
+	if (!AppendIpset(&hfile->ipset, hfile->filename))
+	{
+		ipsetDestroy(&hfile->ipset);
+		return false;
+	}
+	hfile->mod_time=t;
 	return true;
 }
+static bool LoadIpsets(struct ipset_files_head *list)
+{
+	bool bres=true;
+	struct ipset_file *hfile;
+
+	LIST_FOREACH(hfile, list, next)
+	{
+		if (!LoadIpset(hfile))
+			// at least one failed
+			bres=false;
+	}
+	return bres;
+}
 
-bool LoadIncludeIpsets()
+bool LoadAllIpsets()
 {
-	struct desync_profile_list *dpl;
+	return LoadIpsets(&params.ipsets);
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-		if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
-			return false;
-	return true;
-}
-bool LoadExcludeIpsets()
-{
-	struct desync_profile_list *dpl;
-	LIST_FOREACH(dpl, &params.desync_profiles, next)
-		if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
-			return false;
-	return true;
 }
 
-bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
 	char s_ip[40];
 	bool bInSet=false;
@@ -172,24 +181,109 @@ bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_
 	return bInSet;
 }
 
-static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+static bool IpsetsReloadCheck(const struct ipset_collection_head *ipsets)
 {
-	if (!IPSET_EMPTY(ips_exclude))
+	struct ipset_item *item;
+	LIST_FOREACH(item, ipsets, next)
 	{
-		VPRINT("exclude ");
+		if (!LoadIpset(item->hfile))
-		if (SearchIpset(ips_exclude, ipv4, ipv6))
 			return false;
 	}
-	if (!IPSET_EMPTY(ips))
+	return true;
+}
+bool IpsetsReloadCheckForProfile(const struct desync_profile *dp)
+{
+	return IpsetsReloadCheck(&dp->ips_collection) && IpsetsReloadCheck(&dp->ips_collection_exclude);
+}
+
+static bool IpsetCheck_(const struct ipset_collection_head *ips, const struct ipset_collection_head *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+{
+	struct ipset_item *item;
+
+	if (!IpsetsReloadCheck(ips) || !IpsetsReloadCheck(ips_exclude))
+		return false;
+
+	LIST_FOREACH(item, ips_exclude, next)
 	{
-		VPRINT("include ");
+		VPRINT("[%s] exclude ",item->hfile->filename);
-		return SearchIpset(ips, ipv4, ipv6);
+		if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
+			return false;
+	}
+	// old behavior compat: all include lists are empty means check passes
+	if (!ipset_collection_is_empty(ips))
+	{
+		LIST_FOREACH(item, ips, next)
+		{
+			VPRINT("[%s] include ",item->hfile->filename);
+			if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
+				return true;
+		}
+		return false;
 	}
 	return true;
 }
 
-bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
+bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
 {
-	if (!PROFILE_IPSETS_EMPTY(dp)) VPRINT("* ipset check for profile %d\n",dp->n);
+	if (PROFILE_IPSETS_EMPTY(dp)) return true;
-	return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
+	VPRINT("* ipset check for profile %d\n",dp->n);
+	return IpsetCheck_(&dp->ips_collection,&dp->ips_collection_exclude,ipv4,ipv6);
+}
+
+
+static struct ipset_file *RegisterIpset_(struct ipset_files_head *ipsets, struct ipset_collection_head *ips_collection, const char *filename)
+{
+	struct ipset_file *hfile;
+	if (!(hfile=ipset_files_search(ipsets, filename)))
+		if (!(hfile=ipset_files_add(ipsets, filename)))
+			return NULL;
+	if (!ipset_collection_search(ips_collection, filename))
+		if (!ipset_collection_add(ips_collection, hfile))
+			return NULL;
+	return hfile;
+}
+struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename)
+{
+	if (!file_mod_time(filename))
+	{
+		DLOG_ERR("cannot access ipset file '%s'\n",filename);
+		return NULL;
+	}
+	return RegisterIpset_(
+		&params.ipsets,
+		bExclude ? &dp->ips_collection_exclude : &dp->ips_collection,
+		filename);
+}
+
+static const char *dbg_ipset_fill(const ipset *ips)
+{
+	if (ips->ips4)
+		if (ips->ips6)
+			return "ipv4+ipv6";
+		else
+			return "ipv4";
+	else
+		if (ips->ips6)
+			return "ipv6";
+		else
+			return "empty";
+}
+void IpsetsDebug()
+{
+	if (!params.debug) return;
+
+	struct ipset_file *hfile;
+	struct desync_profile_list *dpl;
+	struct ipset_item *ips_item;
+
+	LIST_FOREACH(hfile, &params.ipsets, next)
+		VPRINT("ipset file %s (%s)\n",hfile->filename,dbg_ipset_fill(&hfile->ipset));
+
+	LIST_FOREACH(dpl, &params.desync_profiles, next)
+	{
+		LIST_FOREACH(ips_item, &dpl->dp.ips_collection, next)
+			VPRINT("profile %d include ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
+		LIST_FOREACH(ips_item, &dpl->dp.ips_collection_exclude, next)
+			VPRINT("profile %d exclude ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
+	}
 }

tpws/ipset.h

@@ -5,7 +5,7 @@
 #include "params.h"
 #include "pools.h"
 
-bool LoadIncludeIpsets();
+bool LoadAllIpsets();
-bool LoadExcludeIpsets();
+bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
-bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename);
-bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
+void IpsetsDebug();

tpws/params.c

@@ -145,10 +145,12 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 	struct desync_profile_list *entry = calloc(1,sizeof(struct desync_profile_list));
 	if (!entry) return NULL;
 
-	LIST_INIT(&entry->dp.hostlist_files);
+	LIST_INIT(&entry->dp.hl_collection);
-	LIST_INIT(&entry->dp.hostlist_exclude_files);
+	LIST_INIT(&entry->dp.hl_collection_exclude);
-	entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true;
+	LIST_INIT(&entry->dp.ips_collection);
+	LIST_INIT(&entry->dp.ips_collection_exclude);
 
+	entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true;
 	memcpy(entry->dp.hostspell, "host", 4); // default hostspell
 	entry->dp.hostlist_auto_fail_threshold = HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT;
 	entry->dp.hostlist_auto_fail_time = HOSTLIST_AUTO_FAIL_TIME_DEFAULT;
@@ -167,14 +169,10 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
 }
 static void dp_entry_destroy(struct desync_profile_list *entry)
 {
-	strlist_destroy(&entry->dp.hostlist_files);
+	hostlist_collection_destroy(&entry->dp.hl_collection);
-	strlist_destroy(&entry->dp.hostlist_exclude_files);
+	hostlist_collection_destroy(&entry->dp.hl_collection_exclude);
-	strlist_destroy(&entry->dp.ipset_files);
+	ipset_collection_destroy(&entry->dp.ips_collection);
-	strlist_destroy(&entry->dp.ipset_exclude_files);
+	ipset_collection_destroy(&entry->dp.ips_collection_exclude);
-	StrPoolDestroy(&entry->dp.hostlist_exclude);
-	StrPoolDestroy(&entry->dp.hostlist);
-	ipsetDestroy(&entry->dp.ips);
-	ipsetDestroy(&entry->dp.ips_exclude);
 	HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
 	free(entry);
 }
@@ -187,11 +185,3 @@ void dp_list_destroy(struct desync_profile_list_head *head)
 		dp_entry_destroy(entry);
 	}
 }
-bool dp_list_have_autohostlist(struct desync_profile_list_head *head)
-{
-	struct desync_profile_list *dpl;
-	LIST_FOREACH(dpl, head, next)
-		if (*dpl->dp.hostlist_auto_filename)
-			return true;
-	return false;
-}

tpws/params.h

@@ -48,22 +48,25 @@ struct desync_profile
 
 	bool tamper_start_n,tamper_cutoff_n;
 	unsigned int tamper_start,tamper_cutoff;
-	
+
 	bool filter_ipv4,filter_ipv6;
 	port_filter pf_tcp;
 	uint32_t filter_l7;	// L7_PROTO_* bits
-	ipset ips,ips_exclude;
-	struct str_list_head ipset_files, ipset_exclude_files;
 
-	strpool *hostlist, *hostlist_exclude;
+	// list of pointers to ipsets
-	struct str_list_head hostlist_files, hostlist_exclude_files;
+	struct ipset_collection_head ips_collection, ips_collection_exclude;
-	char hostlist_auto_filename[PATH_MAX];
+
-	int hostlist_auto_fail_threshold, hostlist_auto_fail_time;
+	// list of pointers to hostlist files
-	time_t hostlist_auto_mod_time;
+	struct hostlist_collection_head hl_collection, hl_collection_exclude;
+	// pointer to autohostlist. NULL if no autohostlist for the profile.
+	struct hostlist_file *hostlist_auto;
+	int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
+
 	hostfail_pool *hostlist_auto_fail_counters;
 };
 
-#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
+#define PROFILE_IPSETS_EMPTY(dp) (ipset_collection_is_empty(&dp->ips_collection) && ipset_collection_is_empty(&dp->ips_collection_exclude))
+#define PROFILE_HOSTLISTS_EMPTY(dp) (hostlist_collection_is_empty(&dp->hl_collection) && hostlist_collection_is_empty(&dp->hl_collection_exclude))
 
 struct desync_profile_list {
 	struct desync_profile dp;
@@ -72,7 +75,6 @@ struct desync_profile_list {
 LIST_HEAD(desync_profile_list_head, desync_profile_list);
 struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head);
 void dp_list_destroy(struct desync_profile_list_head *head);
-bool dp_list_have_autohostlist(struct desync_profile_list_head *head);
 
 struct params_s
 {
@@ -112,6 +114,11 @@ struct params_s
 	int ttl_default;
 	char hostlist_auto_debuglog[PATH_MAX];
 
+	// hostlist files with data for all profiles
+	struct hostlist_files_head hostlists;
+	// ipset files with data for all profiles
+	struct ipset_files_head ipsets;
+
 	bool tamper; // any tamper option is set
 	bool tamper_lim; // tamper-start or tamper-cutoff set in any profile
 	struct desync_profile_list_head desync_profiles;

tpws/pools.c

@@ -52,12 +52,6 @@ bool StrPoolAddStr(strpool **pp, const char *s)
 {
 	return StrPoolAddStrLen(pp, s, strlen(s));
 }
-bool StrPoolAddUniqueStr(strpool **pp,const char *s)
-{
-	if (StrPoolCheckStr(*pp,s))
-		return true;
-	return StrPoolAddStr(pp,s);
-}
 
 bool StrPoolCheckStr(strpool *p, const char *s)
 {
@@ -160,6 +154,93 @@ void strlist_destroy(struct str_list_head *head)
 
 
 
+
+struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename)
+{
+	struct hostlist_file *entry = malloc(sizeof(struct hostlist_file));
+	if (entry)
+	{
+		if (!(entry->filename = strdup(filename)))
+		{
+			free(entry);
+			return false;
+		}
+		entry->mod_time=0;
+		entry->hostlist = NULL;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+static void hostlist_files_entry_destroy(struct hostlist_file *entry)
+{
+	if (entry->filename) free(entry->filename);
+	StrPoolDestroy(&entry->hostlist);
+	free(entry);
+}
+void hostlist_files_destroy(struct hostlist_files_head *head)
+{
+	struct hostlist_file *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		hostlist_files_entry_destroy(entry);
+	}
+}
+struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename)
+{
+	struct hostlist_file *hfile;
+
+	LIST_FOREACH(hfile, head, next)
+	{
+		if (!strcmp(hfile->filename,filename))
+			return hfile;
+	}
+	return NULL;
+}
+
+struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile)
+{
+	struct hostlist_item *entry = malloc(sizeof(struct hostlist_item));
+	if (entry)
+	{
+		entry->hfile = hfile;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+void hostlist_collection_destroy(struct hostlist_collection_head *head)
+{
+	struct hostlist_item *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		free(entry);
+	}
+}
+struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename)
+{
+	struct hostlist_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!strcmp(item->hfile->filename,filename))
+			return item;
+	}
+	return NULL;
+}
+bool hostlist_collection_is_empty(const struct hostlist_collection_head *head)
+{
+	const struct hostlist_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (item->hfile->hostlist)
+			return false;
+	}
+	return true;
+}
+
+
 void ipset4Destroy(ipset4 **ipset)
 {
 	ipset4 *elem, *tmp;
@@ -281,3 +362,89 @@ void ipsetPrint(ipset *ipset)
 	ipset4Print(ipset->ips4);
 	ipset6Print(ipset->ips6);
 }
+
+
+struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename)
+{
+	struct ipset_file *entry = malloc(sizeof(struct ipset_file));
+	if (entry)
+	{
+		if (!(entry->filename = strdup(filename)))
+		{
+			free(entry);
+			return false;
+		}
+		entry->mod_time=0;
+		memset(&entry->ipset,0,sizeof(entry->ipset));
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+static void ipset_files_entry_destroy(struct ipset_file *entry)
+{
+	if (entry->filename) free(entry->filename);
+	ipsetDestroy(&entry->ipset);
+	free(entry);
+}
+void ipset_files_destroy(struct ipset_files_head *head)
+{
+	struct ipset_file *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		ipset_files_entry_destroy(entry);
+	}
+}
+struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename)
+{
+	struct ipset_file *hfile;
+
+	LIST_FOREACH(hfile, head, next)
+	{
+		if (!strcmp(hfile->filename,filename))
+			return hfile;
+	}
+	return NULL;
+}
+
+struct ipset_item *ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile)
+{
+	struct ipset_item *entry = malloc(sizeof(struct ipset_item));
+	if (entry)
+	{
+		entry->hfile = hfile;
+		LIST_INSERT_HEAD(head, entry, next);
+	}
+	return entry;
+}
+void ipset_collection_destroy(struct ipset_collection_head *head)
+{
+	struct ipset_item *entry;
+	while ((entry = LIST_FIRST(head)))
+	{
+		LIST_REMOVE(entry, next);
+		free(entry);
+	}
+}
+struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename)
+{
+	struct ipset_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!strcmp(item->hfile->filename,filename))
+			return item;
+	}
+	return NULL;
+}
+bool ipset_collection_is_empty(const struct ipset_collection_head *head)
+{
+	const struct ipset_item *item;
+
+	LIST_FOREACH(item, head, next)
+	{
+		if (!IPSET_EMPTY(&item->hfile->ipset))
+			return false;
+	}
+	return true;
+}

tpws/pools.h

@@ -19,7 +19,6 @@ typedef struct strpool {
 
 void StrPoolDestroy(strpool **pp);
 bool StrPoolAddStr(strpool **pp,const char *s);
-bool StrPoolAddUniqueStr(strpool **pp,const char *s);
 bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen);
 bool StrPoolCheckStr(strpool *p,const char *s);
 
@@ -29,7 +28,6 @@ struct str_list {
 };
 LIST_HEAD(str_list_head, str_list);
 
-
 typedef struct hostfail_pool {
     char *str;		/* key */
     int counter;	/* value */
@@ -49,6 +47,30 @@ bool strlist_add(struct str_list_head *head, const char *filename);
 void strlist_destroy(struct str_list_head *head);
 
 
+
+struct hostlist_file {
+	char *filename;
+	time_t mod_time;
+	strpool *hostlist;
+	LIST_ENTRY(hostlist_file) next;
+};
+LIST_HEAD(hostlist_files_head, hostlist_file);
+
+struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename);
+void hostlist_files_destroy(struct hostlist_files_head *head);
+struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename);
+
+struct hostlist_item {
+	struct hostlist_file *hfile;
+	LIST_ENTRY(hostlist_item) next;
+};
+LIST_HEAD(hostlist_collection_head, hostlist_item);
+struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile);
+void hostlist_collection_destroy(struct hostlist_collection_head *head);
+struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename);
+bool hostlist_collection_is_empty(const struct hostlist_collection_head *head);
+
+
 typedef struct ipset4 {
 	struct cidr4 cidr;	/* key */
 	UT_hash_handle hh;	/* makes this structure hashable */
@@ -85,3 +107,26 @@ void ipset6Print(ipset6 *ipset);
 
 void ipsetDestroy(ipset *ipset);
 void ipsetPrint(ipset *ipset);
+
+
+struct ipset_file {
+	char *filename;
+	time_t mod_time;
+	ipset ipset;
+	LIST_ENTRY(ipset_file) next;
+};
+LIST_HEAD(ipset_files_head, ipset_file);
+
+struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename);
+void ipset_files_destroy(struct ipset_files_head *head);
+struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename);
+
+struct ipset_item {
+	struct ipset_file *hfile;
+	LIST_ENTRY(ipset_item) next;
+};
+LIST_HEAD(ipset_collection_head, ipset_item);
+struct ipset_item * ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile);
+void ipset_collection_destroy(struct ipset_collection_head *head);
+struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename);
+bool ipset_collection_is_empty(const struct ipset_collection_head *head);

tpws/tamper.c

@@ -24,40 +24,36 @@ static bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7)
 		(l7proto==TLS && (filter_l7 & L7_PROTO_TLS));
 }
 
-static bool dp_match_l3l4(struct desync_profile *dp, const struct sockaddr *dest)
-{
-	return  ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
-		pf_in_range(saport(dest), &dp->pf_tcp) &&
-		IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
-}
-static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
-{
-	return	!PROFILE_IPSETS_EMPTY(dp) &&
-		((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
-}
 static bool dp_match(struct desync_profile *dp, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
 {
-	// impossible case, hard filter
+	if (!HostlistsReloadCheckForProfile(dp)) return false;
-	// impossible check avoids relatively slow ipset search
-	if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,dest))
-	{
-		// soft filter
-		if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
-			return false;
 
-		// autohostlist profile matching l3/l4 filter always win
+	if ((dest->sa_family==AF_INET && !dp->filter_ipv4) || (dest->sa_family==AF_INET6 && !dp->filter_ipv6))
-		if (*dp->hostlist_auto_filename) return true;
+		// L3 filter does not match
+		return false;
+	if (!pf_in_range(saport(dest), &dp->pf_tcp))
+		// L4 filter does not match
+		return false;
+	if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
+		// L7 filter does not match
+		return false;
+	if (!dp->hostlist_auto && !hostname && !PROFILE_HOSTLISTS_EMPTY(dp))
+		// avoid cpu consuming ipset check. profile cannot win if regular hostlists are present without auto hostlist and hostname is unknown.
+		return false;
+	if (!IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL))
+		// target ip does not match
+		return false;
+
+	// autohostlist profile matching l3/l4 filter always win
+	if (dp->hostlist_auto) return true;
+
+	if (PROFILE_HOSTLISTS_EMPTY(dp))
+		// profile without hostlist filter wins
+		return true;
+	else if (hostname)
+		// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
+		return HostlistCheck(dp, hostname, NULL, true);
 
-		if (dp->hostlist || dp->hostlist_exclude)
-		{
-			// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
-			if (hostname)
-				return HostlistCheck(dp, hostname, NULL);
-		}
-		else
-			// profile without hostlist filter wins
-			return true;
-	}
 	return false;
 }
 static struct desync_profile *dp_find(struct desync_profile_list_head *head, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
@@ -181,10 +177,10 @@ void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment,
 			VPRINT("desync profile changed by revealed l7 protocol or hostname !\n");
 	}
 
-	if (bDiscoveredHostname && *ctrack->dp->hostlist_auto_filename)
+	if (bDiscoveredHostname && ctrack->dp->hostlist_auto)
 	{
 		bool bHostExcluded;
-		if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded))
+		if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded, false))
 		{
 			ctrack->b_ah_check = !bHostExcluded;
 			VPRINT("Not acting on this request\n");
@@ -415,21 +411,21 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
 		
 		VPRINT("auto hostlist (profile %d) : rechecking %s to avoid duplicates\n", dp->n, hostname);
 		bool bExcluded=false;
-		if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
+		if (!HostlistCheck(dp, hostname, &bExcluded, false) && !bExcluded)
 		{
-			VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
+			VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto->filename);
-			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
+			HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto->filename);
-			if (!StrPoolAddStr(&dp->hostlist, hostname))
+			if (!StrPoolAddStr(&dp->hostlist_auto->hostlist, hostname))
 			{
 				DLOG_ERR("StrPoolAddStr out of memory\n");
 				return;
 			}
-			if (!append_to_list_file(dp->hostlist_auto_filename, hostname))
+			if (!append_to_list_file(dp->hostlist_auto->filename, hostname))
 			{
 				DLOG_PERROR("write to auto hostlist:");
 				return;
 			}
-			dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
+			dp->hostlist_auto->mod_time = file_mod_time(dp->hostlist_auto->filename);
 		}
 		else
 		{

tpws/tpws.c

@@ -43,25 +43,9 @@
 
 struct params_s params;
 
-bool bHup = false;
 static void onhup(int sig)
 {
 	printf("HUP received !\n");
-	printf("Will reload hostlists and ipsets on next request (if any)\n");
-	bHup = true;
-}
-// should be called in normal execution
-void dohup(void)
-{
-	if (bHup)
-	{
-		if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
-		{
-			// what will we do without hostlist or ipset ?? sure, gonna die
-			exit(1);
-		}
-		bHup = false;
-	}
 }
 
 static void onusr2(int sig)
@@ -230,6 +214,9 @@ static void exithelp(void)
 static void cleanup_params(void)
 {
 	dp_list_destroy(&params.desync_profiles);
+
+	hostlist_files_destroy(&params.hostlists);
+	ipset_files_destroy(&params.ipsets);
 }
 static void exithelp_clean(void)
 {
@@ -367,6 +354,10 @@ void parse_params(int argc, char *argv[])
 #if defined(__OpenBSD__) || defined(__APPLE__)
 	params.pf_enable = true; // OpenBSD and MacOS have no other choice
 #endif
+
+	LIST_INIT(&params.hostlists);
+	LIST_INIT(&params.ipsets);
+
 	if (can_drop_root())
 	{
 	    params.uid = params.gid = 0x7FFFFFFF; // default uid:gid
@@ -726,29 +717,29 @@ void parse_params(int argc, char *argv[])
 			params.tamper = true;
 			break;
 		case 36: /* hostlist */
-			if (!strlist_add(&dp->hostlist_files, optarg))
+			if (!RegisterHostlist(dp, false, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
 				exit_clean(1);
 			}
 			params.tamper = true;
 			break;
 		case 37: /* hostlist-exclude */
-			if (!strlist_add(&dp->hostlist_exclude_files, optarg))
+			if (!RegisterHostlist(dp, true, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
 				exit_clean(1);
 			}
 			params.tamper = true;
 			break;
 		case 38: /* hostlist-auto */
-			if (*dp->hostlist_auto_filename)
+			if (dp->hostlist_auto)
 			{
 				DLOG_ERR("only one auto hostlist per profile is supported\n");
 				exit_clean(1);
 			}
 			{
-				FILE *F = fopen(optarg,"a+t");
+				FILE *F = fopen(optarg,"a+b");
 				if (!F)
 				{
 					DLOG_ERR("cannot create %s\n", optarg);
@@ -764,13 +755,11 @@ void parse_params(int argc, char *argv[])
 				if (params.droproot && chown(optarg, params.uid, -1))
 					DLOG_ERR("could not chown %s. auto hostlist file may not be writable after privilege drop\n", optarg);
 			}
-			if (!strlist_add(&dp->hostlist_files, optarg))
+			if (!(dp->hostlist_auto=RegisterHostlist(dp, false, optarg)))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register hostlist '%s'\n", optarg);
 				exit_clean(1);
 			}
-			strncpy(dp->hostlist_auto_filename, optarg, sizeof(dp->hostlist_auto_filename));
-			dp->hostlist_auto_filename[sizeof(dp->hostlist_auto_filename) - 1] = '\0';
 			params.tamper = true; // need to detect blocks and update autohostlist. cannot just slice.
 			break;
 		case 39: /* hostlist-auto-fail-threshold */
@@ -979,17 +968,17 @@ void parse_params(int argc, char *argv[])
 			}
 			break;
 		case 60: /* ipset */
-			if (!strlist_add(&dp->ipset_files, optarg))
+			if (!RegisterIpset(dp, false, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register ipset '%s'\n", optarg);
 				exit_clean(1);
 			}
 			params.tamper = true;
 			break;
 		case 61: /* ipset-exclude */
-			if (!strlist_add(&dp->ipset_exclude_files, optarg))
+			if (!RegisterIpset(dp, true, optarg))
 			{
-				DLOG_ERR("strlist_add failed\n");
+				DLOG_ERR("failed to register ipset '%s'\n", optarg);
 				exit_clean(1);
 			}
 			params.tamper = true;
@@ -1062,7 +1051,6 @@ void parse_params(int argc, char *argv[])
 		dp = &dpl->dp;
 		if (dp->split_tls==tlspos_none && dp->split_pos) dp->split_tls=tlspos_pos;
 		if (dp->split_http_req==httpreqpos_none && dp->split_pos) dp->split_http_req=httpreqpos_pos;
-		if (*dp->hostlist_auto_filename) dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
 		if (params.skip_nodelay && (dp->split_tls || dp->split_http_req || dp->split_pos))
 		{
 			DLOG_ERR("Cannot split with --skip-nodelay\n");
@@ -1070,26 +1058,21 @@ void parse_params(int argc, char *argv[])
 		}
 	}
 
-	if (!LoadIncludeHostLists())
+	if (!LoadAllHostLists())
 	{
-		DLOG_ERR("Include hostlist load failed\n");
+		DLOG_ERR("hostlists load failed\n");
 		exit_clean(1);
 	}
-	if (!LoadExcludeHostLists())
+	if (!LoadAllIpsets())
 	{
-		DLOG_ERR("Exclude hostlist load failed\n");
+		DLOG_ERR("ipset load failed\n");
-		exit_clean(1);
-	}
-	if (!LoadIncludeIpsets())
-	{
-		DLOG_ERR("Include ipset load failed\n");
-		exit_clean(1);
-	}
-	if (!LoadExcludeIpsets())
-	{
-		DLOG_ERR("Exclude ipset load failed\n");
 		exit_clean(1);
 	}
+
+	VPRINT("\nlists summary:\n");
+	HostlistsDebug();
+	IpsetsDebug();
+	VPRINT("\n");
 }
 
 

tpws/tpws_conn.c

@@ -1469,8 +1469,6 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct)
 			break;
 		}
 
-		dohup();
-
 		for (i = 0; i < num_events; i++)
 		{
 			conn = (tproxy_conn_t*)events[i].data.ptr;