diff --git a/binaries/aarch64/nfqws b/binaries/aarch64/nfqws index 56d4d72..a75627b 100755 Binary files a/binaries/aarch64/nfqws and b/binaries/aarch64/nfqws differ diff --git a/binaries/aarch64/tpws b/binaries/aarch64/tpws index b08bef2..62bbf87 100755 Binary files a/binaries/aarch64/tpws and b/binaries/aarch64/tpws differ diff --git a/binaries/arm/nfqws b/binaries/arm/nfqws index ebec8a6..fb777b0 100755 Binary files a/binaries/arm/nfqws and b/binaries/arm/nfqws differ diff --git a/binaries/arm/tpws b/binaries/arm/tpws index 6780b0f..c8bb041 100755 Binary files a/binaries/arm/tpws and b/binaries/arm/tpws differ diff --git a/binaries/freebsd-x64/dvtws b/binaries/freebsd-x64/dvtws index 535a050..c46a5ef 100755 Binary files a/binaries/freebsd-x64/dvtws and b/binaries/freebsd-x64/dvtws differ diff --git a/binaries/freebsd-x64/tpws b/binaries/freebsd-x64/tpws index af4530a..c7a6b69 100755 Binary files a/binaries/freebsd-x64/tpws and b/binaries/freebsd-x64/tpws differ diff --git a/binaries/mac64/tpws b/binaries/mac64/tpws index e2e26b0..311f62a 100755 Binary files a/binaries/mac64/tpws and b/binaries/mac64/tpws differ diff --git a/binaries/mips32r1-lsb/nfqws b/binaries/mips32r1-lsb/nfqws index 3249cc5..08217c6 100755 Binary files a/binaries/mips32r1-lsb/nfqws and b/binaries/mips32r1-lsb/nfqws differ diff --git a/binaries/mips32r1-lsb/tpws b/binaries/mips32r1-lsb/tpws index 75b5409..e61d2f9 100755 Binary files a/binaries/mips32r1-lsb/tpws and b/binaries/mips32r1-lsb/tpws differ diff --git a/binaries/mips32r1-msb/nfqws b/binaries/mips32r1-msb/nfqws index fd46fbc..756d313 100755 Binary files a/binaries/mips32r1-msb/nfqws and b/binaries/mips32r1-msb/nfqws differ diff --git a/binaries/mips32r1-msb/tpws b/binaries/mips32r1-msb/tpws index c460be5..66eb71d 100755 Binary files a/binaries/mips32r1-msb/tpws and b/binaries/mips32r1-msb/tpws differ diff --git a/binaries/mips64r2-msb/nfqws b/binaries/mips64r2-msb/nfqws index 9a9d15d..9127bab 100755 Binary files a/binaries/mips64r2-msb/nfqws and b/binaries/mips64r2-msb/nfqws differ diff --git a/binaries/mips64r2-msb/tpws b/binaries/mips64r2-msb/tpws index 9fe09e2..c753bd9 100755 Binary files a/binaries/mips64r2-msb/tpws and b/binaries/mips64r2-msb/tpws differ diff --git a/binaries/ppc/nfqws b/binaries/ppc/nfqws index 7d5fd8a..edafc55 100755 Binary files a/binaries/ppc/nfqws and b/binaries/ppc/nfqws differ diff --git a/binaries/ppc/tpws b/binaries/ppc/tpws index 62f3e7c..0e4acab 100755 Binary files a/binaries/ppc/tpws and b/binaries/ppc/tpws differ diff --git a/binaries/x86/nfqws b/binaries/x86/nfqws index 774e8bf..53c17c6 100755 Binary files a/binaries/x86/nfqws and b/binaries/x86/nfqws differ diff --git a/binaries/x86/tpws b/binaries/x86/tpws index cfa0cb0..d35d305 100755 Binary files a/binaries/x86/tpws and b/binaries/x86/tpws differ diff --git a/binaries/x86_64/nfqws b/binaries/x86_64/nfqws index cd4e29d..0a30f3d 100755 Binary files a/binaries/x86_64/nfqws and b/binaries/x86_64/nfqws differ diff --git a/binaries/x86_64/tpws b/binaries/x86_64/tpws index f1238fd..301b162 100755 Binary files a/binaries/x86_64/tpws and b/binaries/x86_64/tpws differ diff --git a/binaries/x86_64/tpws_wsl.tgz b/binaries/x86_64/tpws_wsl.tgz index f95ff9c..0e4f3f8 100644 Binary files a/binaries/x86_64/tpws_wsl.tgz and b/binaries/x86_64/tpws_wsl.tgz differ diff --git a/docs/readme.eng.md b/docs/readme.eng.md index a9c98f7..f69cf33 100644 --- a/docs/readme.eng.md +++ b/docs/readme.eng.md @@ -796,6 +796,10 @@ may start to break the website. This situation can only be controlled manually. Remove undesired domain from the autohostlist file, restart nfqws/tpws or send them SIGHUP. Use exclude hostlist to prevent further auto additions. +It's possible to use one auto hostlist with multiple processes. All processes check for file modification time. +If a process modified autohostlist, all others will reread it automatically. +All processes must run with the same uid. + If zapret scripts are used then autohostlist is `ipset/zapret-hosts-auto.txt` and exlude list is `ipset/zapret-hosts-user-exclude.txt`. autohostlist mode includes hostlist mode. You can use `ipset/zapret-hosts-user.txt`. diff --git a/docs/readme.txt b/docs/readme.txt index 7274b34..cc0edd3 100644 --- a/docs/readme.txt +++ b/docs/readme.txt @@ -1041,6 +1041,10 @@ nfqws и tpws могут сечь варианты 1-3, 4 они не распо В лог заносятся только основные события, ведущие к занесению хоста в лист. По логу можно понять как избежать ложных срабатываний и подходит ли вообще вам этот режим. +Возможно использование одного auto листа с множеством процессов. Все процессы проверяют +время изменения файла. Если другой процесс туда что-то записал, то остальные перечитают все include листы. +Все процессы должны работать под одним uid, чтобы избежать проблем с доступом к файлу. + Скрипты zapret ведут autohostlist в ipset/zapret-hosts-auto.txt. install_easy.sh при апгрейде zapret сохраняет этот файл. Режим autohostlist включает в себя режим hostlist. diff --git a/nfq/desync.c b/nfq/desync.c index 3eb74f8..7cacde6 100644 --- a/nfq/desync.c +++ b/nfq/desync.c @@ -235,7 +235,7 @@ static void auto_hostlist_failed(const char *hostname) DLOG("auto hostlist : rechecking %s to avoid duplicates\n", hostname); bool bExcluded=false; - if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded) + if (!HostlistCheck(hostname, &bExcluded) && !bExcluded) { DLOG("auto hostlist : adding %s\n", hostname); HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname); @@ -249,6 +249,7 @@ static void auto_hostlist_failed(const char *hostname) perror("write to auto hostlist:"); return; } + params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename); } else { @@ -615,7 +616,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, { bool bExcluded; DLOG("hostname: %s\n",host) - if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) + if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(host, &bExcluded)) { DLOG("not applying tampering to this request\n") if (ctrack) @@ -1074,7 +1075,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout, { DLOG("hostname: %s\n",host) bool bExcluded; - if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded)) + if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(host, &bExcluded)) { DLOG("not applying tampering to this request\n") if (!bExcluded && *params.hostlist_auto_filename && ctrack) diff --git a/nfq/helpers.c b/nfq/helpers.c index 72b902d..a2ca4b7 100644 --- a/nfq/helpers.c +++ b/nfq/helpers.c @@ -6,6 +6,7 @@ #include #include #include +#include void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit) { @@ -256,3 +257,9 @@ int fprint_localtime(FILE *F) localtime_r(&now,&t); return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec); } + +time_t file_mod_time(const char *filename) +{ + struct stat st; + return stat(filename,&st)==-1 ? 0 : st.st_mtime; +} diff --git a/nfq/helpers.h b/nfq/helpers.h index c2d8762..e1f6b22 100644 --- a/nfq/helpers.h +++ b/nfq/helpers.h @@ -45,3 +45,5 @@ bool parse_hex_str(const char *s, uint8_t *pbuf, size_t *size); void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize); int fprint_localtime(FILE *F); + +time_t file_mod_time(const char *filename); diff --git a/nfq/hostlist.c b/nfq/hostlist.c index 9c81efb..c9163f5 100644 --- a/nfq/hostlist.c +++ b/nfq/hostlist.c @@ -2,7 +2,7 @@ #include "hostlist.h" #include "gzip.h" #include "params.h" - +#include "helpers.h" // inplace tolower() and add to pool static bool addpool(strpool **hostlist, char **s, const char *end) @@ -132,7 +132,7 @@ bool SearchHostList(strpool *hostlist, const char *host) } // return : true = apply fooling, false = do not apply -bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded) +static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded) { if (excluded) *excluded = false; if (hostlist_exclude) @@ -151,3 +151,36 @@ bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *hos } return true; } + +// return : true = apply fooling, false = do not apply +bool HostlistCheck(const char *host, bool *excluded) +{ + if (*params.hostlist_auto_filename) + { + time_t t = file_mod_time(params.hostlist_auto_filename); + if (t!=params.hostlist_auto_mod_time) + { + printf("Autohostlist was modified by another process. Reloading include hostslist.\n"); + if (!LoadIncludeHostLists()) + { + // what will we do without hostlist ?? sure, gonna die + exit(1); + } + params.hostlist_auto_mod_time = t; + } + } + return HostlistCheck_(params.hostlist, params.hostlist_exclude, host, excluded); +} + +bool LoadIncludeHostLists() +{ + if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files)) + return false; + if (*params.hostlist_auto_filename) + params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename); + return true; +} +bool LoadExcludeHostLists() +{ + return LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files); +} diff --git a/nfq/hostlist.h b/nfq/hostlist.h index 5940ac6..ba0e34b 100644 --- a/nfq/hostlist.h +++ b/nfq/hostlist.h @@ -5,7 +5,9 @@ bool AppendHostList(strpool **hostlist, char *filename); bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list); +bool LoadIncludeHostLists(); +bool LoadExcludeHostLists(); bool NonEmptyHostlist(strpool **hostlist); bool SearchHostList(strpool *hostlist, const char *host); // return : true = apply fooling, false = do not apply -bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded); +bool HostlistCheck(const char *host, bool *excluded); \ No newline at end of file diff --git a/nfq/nfqws.c b/nfq/nfqws.c index 8c97e73..b5dfec3 100644 --- a/nfq/nfqws.c +++ b/nfq/nfqws.c @@ -56,8 +56,7 @@ static void dohup(void) { if (bHup) { - if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files) || - !LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files)) + if (!LoadIncludeHostLists() || !LoadExcludeHostLists()) { // what will we do without hostlist ?? sure, gonna die exit(1); @@ -1232,13 +1231,13 @@ int main(int argc, char **argv) } #endif - if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files)) + if (!LoadIncludeHostLists()) { fprintf(stderr, "Include hostlist load failed\n"); exit_clean(1); } if (*params.hostlist_auto_filename) NonEmptyHostlist(¶ms.hostlist); - if (!LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files)) + if (!LoadExcludeHostLists()) { fprintf(stderr, "Exclude hostlist load failed\n"); exit_clean(1); diff --git a/nfq/params.h b/nfq/params.h index cc01667..026e8bb 100644 --- a/nfq/params.h +++ b/nfq/params.h @@ -10,6 +10,7 @@ #include #include #include +#include #define TLS_PARTIALS_ENABLE true @@ -68,6 +69,7 @@ struct params_s struct str_list_head hostlist_files, hostlist_exclude_files; char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX]; int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold; + time_t hostlist_auto_mod_time; hostfail_pool *hostlist_auto_fail_counters; unsigned int ctrack_t_syn, ctrack_t_est, ctrack_t_fin, ctrack_t_udp; diff --git a/tpws/helpers.c b/tpws/helpers.c index 4ac46d0..feb85b9 100644 --- a/tpws/helpers.c +++ b/tpws/helpers.c @@ -9,6 +9,7 @@ #include #include #include +#include char *strncasestr(const char *s,const char *find, size_t slen) { @@ -222,3 +223,9 @@ int fprint_localtime(FILE *F) localtime_r(&now,&t); return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec); } + +time_t file_mod_time(const char *filename) +{ + struct stat st; + return stat(filename,&st)==-1 ? 0 : st.st_mtime; +} diff --git a/tpws/helpers.h b/tpws/helpers.h index 00e0522..00a1bc4 100644 --- a/tpws/helpers.h +++ b/tpws/helpers.h @@ -6,6 +6,7 @@ #include #include #include +#include char *strncasestr(const char *s,const char *find, size_t slen); @@ -44,3 +45,5 @@ static inline void phton16(uint8_t *p, uint16_t v) { } int fprint_localtime(FILE *F); + +time_t file_mod_time(const char *filename); diff --git a/tpws/hostlist.c b/tpws/hostlist.c index 9c81efb..c9163f5 100644 --- a/tpws/hostlist.c +++ b/tpws/hostlist.c @@ -2,7 +2,7 @@ #include "hostlist.h" #include "gzip.h" #include "params.h" - +#include "helpers.h" // inplace tolower() and add to pool static bool addpool(strpool **hostlist, char **s, const char *end) @@ -132,7 +132,7 @@ bool SearchHostList(strpool *hostlist, const char *host) } // return : true = apply fooling, false = do not apply -bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded) +static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded) { if (excluded) *excluded = false; if (hostlist_exclude) @@ -151,3 +151,36 @@ bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *hos } return true; } + +// return : true = apply fooling, false = do not apply +bool HostlistCheck(const char *host, bool *excluded) +{ + if (*params.hostlist_auto_filename) + { + time_t t = file_mod_time(params.hostlist_auto_filename); + if (t!=params.hostlist_auto_mod_time) + { + printf("Autohostlist was modified by another process. Reloading include hostslist.\n"); + if (!LoadIncludeHostLists()) + { + // what will we do without hostlist ?? sure, gonna die + exit(1); + } + params.hostlist_auto_mod_time = t; + } + } + return HostlistCheck_(params.hostlist, params.hostlist_exclude, host, excluded); +} + +bool LoadIncludeHostLists() +{ + if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files)) + return false; + if (*params.hostlist_auto_filename) + params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename); + return true; +} +bool LoadExcludeHostLists() +{ + return LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files); +} diff --git a/tpws/hostlist.h b/tpws/hostlist.h index 5940ac6..ba0e34b 100644 --- a/tpws/hostlist.h +++ b/tpws/hostlist.h @@ -5,7 +5,9 @@ bool AppendHostList(strpool **hostlist, char *filename); bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list); +bool LoadIncludeHostLists(); +bool LoadExcludeHostLists(); bool NonEmptyHostlist(strpool **hostlist); bool SearchHostList(strpool *hostlist, const char *host); // return : true = apply fooling, false = do not apply -bool HostlistCheck(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded); +bool HostlistCheck(const char *host, bool *excluded); \ No newline at end of file diff --git a/tpws/params.h b/tpws/params.h index b1bfc2c..faa6f84 100644 --- a/tpws/params.h +++ b/tpws/params.h @@ -5,9 +5,10 @@ #include #include #include +#include #include "pools.h" -#define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT 2 +#define HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT 3 #define HOSTLIST_AUTO_FAIL_TIME_DEFAULT 60 enum splithttpreq { split_none = 0, split_method, split_host }; @@ -59,6 +60,7 @@ struct params_s struct str_list_head hostlist_files, hostlist_exclude_files; char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX]; int hostlist_auto_fail_threshold, hostlist_auto_fail_time; + time_t hostlist_auto_mod_time; hostfail_pool *hostlist_auto_fail_counters; bool tamper_start_n,tamper_cutoff_n; diff --git a/tpws/tamper.c b/tpws/tamper.c index e93b5ca..982fe67 100644 --- a/tpws/tamper.c +++ b/tpws/tamper.c @@ -64,7 +64,7 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si bHaveHost = true; VPRINT("Requested Host is : %s", Host) for(pc = Host; *pc; pc++) *pc=tolower(*pc); - bBypass = !HostlistCheck(params.hostlist, params.hostlist_exclude, Host, &bHostExcluded); + bBypass = !HostlistCheck(Host, &bHostExcluded); } if (!bBypass) { @@ -228,7 +228,7 @@ void tamper_out(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,si { VPRINT("hostname: %s",Host) bHaveHost = true; - bBypass = !HostlistCheck(params.hostlist, params.hostlist_exclude, Host, &bHostExcluded); + bBypass = !HostlistCheck(Host, &bHostExcluded); } if (bBypass) { @@ -307,7 +307,7 @@ static void auto_hostlist_failed(const char *hostname) VPRINT("auto hostlist : rechecking %s to avoid duplicates", hostname); bool bExcluded=false; - if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded) + if (!HostlistCheck(hostname, &bExcluded) && !bExcluded) { VPRINT("auto hostlist : adding %s", hostname); HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname); @@ -321,6 +321,7 @@ static void auto_hostlist_failed(const char *hostname) perror("write to auto hostlist:"); return; } + params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename); } else { diff --git a/tpws/tpws.c b/tpws/tpws.c index a6ff607..14c1ed0 100644 --- a/tpws/tpws.c +++ b/tpws/tpws.c @@ -54,8 +54,7 @@ void dohup(void) { if (bHup) { - if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files) || - !LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files)) + if (!LoadIncludeHostLists() || !LoadExcludeHostLists()) { // what will we do without hostlist ?? sure, gonna die exit(1); @@ -717,13 +716,14 @@ void parse_params(int argc, char *argv[]) exit_clean(1); } - if (!LoadHostLists(¶ms.hostlist, ¶ms.hostlist_files)) + if (*params.hostlist_auto_filename) params.hostlist_auto_mod_time = file_mod_time(params.hostlist_auto_filename); + if (!LoadIncludeHostLists()) { fprintf(stderr, "Include hostlist load failed\n"); exit_clean(1); } if (*params.hostlist_auto_filename) NonEmptyHostlist(¶ms.hostlist); - if (!LoadHostLists(¶ms.hostlist_exclude, ¶ms.hostlist_exclude_files)) + if (!LoadExcludeHostLists()) { fprintf(stderr, "Exclude hostlist load failed\n"); exit_clean(1);