From d78705ea746780f67c960eb35e8333344b3a3d23 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Thu, 25 Jul 2024 09:51:23 +0300 Subject: [PATCH 01/20] Fix compilation issues --- Makefile | 13 +++++++++---- README.md | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d84f590..988ffc9 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,11 @@ LD := gcc CFLAGS:=-Wall -Wpedantic -Wno-unused-variable -I$(DEPSDIR)/include -Os LDFLAGS:=-L$(DEPSDIR)/lib -static +LIBNFNETLINK_CFLAGS := -I$(DEPSDIR)/include +LIBNFNETLINK_LIBS := -L$(DEPSDIR)/lib +LIBMNL_CFLAGS := -I$(DEPSDIR)/include +LIBMNL_LIBS := -L$(DEPSDIR)/lib + # PREFIX is environment variable, if not set default to /usr/local ifeq ($(PREFIX),) PREFIX := /usr/local @@ -13,7 +18,7 @@ else PREFIX := $(DESTDIR) endif -export CC LD CFLAGS LDFLAGS +export CC LD CFLAGS LDFLAGS LIBNFNETLINK_CFLAGS LIBNFNETLINK_LIBS LIBMNL_CFLAGS LIBMNL_LIBS APP:=$(BUILD_DIR)/youtubeUnblock @@ -43,17 +48,17 @@ prepare_dirs: mkdir -p $(DEPSDIR) $(LIBNFNETLINK): - cd deps/libnfnetlink && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) + cd deps/libnfnetlink && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) --enable-static --disable-shared $(MAKE) -C deps/libnfnetlink $(MAKE) install -C deps/libnfnetlink $(LIBMNL): - cd deps/libmnl && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) + cd deps/libmnl && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) --enable-static --disable-shared $(MAKE) -C deps/libmnl $(MAKE) install -C deps/libmnl $(LIBNETFILTER_QUEUE): $(LIBNFNETLINK) $(LIBMNL) - cd deps/libnetfilter_queue && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) + cd deps/libnetfilter_queue && ./autogen.sh && ./configure --prefix=$(DEPSDIR) $(if $(CROSS_COMPILE_PLATFORM),--host=$(CROSS_COMPILE_PLATFORM),) --enable-static --disable-shared $(MAKE) -C deps/libnetfilter_queue $(MAKE) install -C deps/libnetfilter_queue diff --git a/README.md b/README.md index 8702649..f05fff3 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ You may read further in an [yt-dlp issue page](https://github.com/yt-dlp/yt-dlp/ When the packet is joining the queue, the application checks sni payload to be googlevideo (right how the DPIs do), segmentates/fragmentates (both TCP and IP fragmentation techniques are supported) and posts the packet. Note that it is impossible to post two fragmented packets from one netfilter queue verdict. Instead, the application drops an original packet and makes another linux raw socket to post the packets in the network. To escape infinity loops the socket marks outgoing packets and the application automatically accepts it. ## Usage: +Before compilation make sure `gcc`, `make`, `autoconf`, `automake`, `pkg-config` and `libtool` is installed. For Fedora `glibc-static` should be installed as well. Compile with `make`. Install with `make install`. The package include libnetfilter_queue, libnfnetlink and libmnl as static dependencies. The package requires linux-headers and kernel built with netfilter nfqueue support. You should also configure iptables for this to start working: From ce0efaeb7b24e8910b286ab1d38fae9594d3fa28 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Wed, 31 Jul 2024 09:52:22 +0300 Subject: [PATCH 02/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f05fff3..1896b90 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # youtubeUnblock Bypasses Googlevideo detection systems that relies on SNI. The package is for Linux only. -For Windows use [GoodbyeDPI from VladikSS](https://github.com/ValdikSS/GoodbyeDPI) (Users points out that working options for it are `goodbyedpi.exe -6 --frag-by-sni`.) The same behavior is also implemented in [zapret package for linux](https://github.com/bol-van/zapret). +For Windows use [GoodbyeDPI from ValdikSS](https://github.com/ValdikSS/GoodbyeDPI) (Users points out that working options for it are `goodbyedpi.exe -6 --frag-by-sni`.) The same behavior is also implemented in [zapret package for linux](https://github.com/bol-van/zapret). ## How it works: Lets look from the DPIses side of view: All they have is ip and tcp information, higher-level data is encrypted. So from the IP header only IP address might be helpful for them. In tcp here is basically nothing. So they may handle IP addresses and process it. What's wrong? Google servers are on the way: It is very hard to handle all that infrastracture. One server may host multiple websites and it is very bad if them block, say Google Search trying to block googlevideo. But even if googlevideo servers have their own ip for only googlevideo purposes, here is a problem about how large is Google infrastracture and how much servers are here. The DPIs can't even parse normally all the servers, because each video may live on it's cache server. So what's else? Let's take a look at a TLS level. All information here is encrypted. All... Except hello messages! They are used to initialize handshake connections and hold tons of helpful information. If we talk about TLS v1.3, it is optimized to transfer as less information as possible unencrypted. But here is only one thing that may point us which domain the user wants to connect, the SNI extension. It transfers all domain names unencrypted. Exactly what we need! And DPIs may use this thing to detect google video connections and slow down them (In fact they are corrupting a tcp connection with bad packets). From 0cb34be25fe4283db5d7d0f1fa6551b562460d1f Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Wed, 31 Jul 2024 13:51:51 +0300 Subject: [PATCH 03/20] (feature) Solves #7 --- Makefile | 2 +- README.md | 2 +- youtubeUnblock.service | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 988ffc9..13ac831 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ install: all uninstall: rm $(PREFIX)/bin/youtubeUnblock rm $(PREFIX)/lib/systemd/system/youtubeUnblock.service - systemctl disable youtubeUnblock.service + -systemctl disable youtubeUnblock.service clean: rm -rf $(BUILD_DIR) diff --git a/README.md b/README.md index 1896b90..01a662b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Here iptables serves every tcp packet, destinating port 443 for this userspace p Run an application with `youtubeUnblock 537` where `537` stands for the queue-num (must be the same as in the iptables rule). -Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). +Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). Please, note that systemd will configure iptables manually. If you have troubles with it, delete ExecStartPre and ExecStop from youtubeUnblock.service and configure iptables manually (may be a useful case for nftables). Also DNS over HTTPS (DOH) is preferred for additional anonimity. diff --git a/youtubeUnblock.service b/youtubeUnblock.service index 672dc6a..fba168d 100644 --- a/youtubeUnblock.service +++ b/youtubeUnblock.service @@ -5,7 +5,9 @@ Description=youtubeUnblock StandardError=journal StandardOutput=journal StandardInput=null +ExecStartPre=iptables -t mangle -A OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass ExecStart=$(PREFIX)/bin/youtubeUnblock 537 +ExecStop=iptables -t mangle -D OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass [Install] WantedBy=multi-user.target From 625ac79825c95d4213bf645f4fe1ee0a35ea7b3d Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Wed, 31 Jul 2024 14:09:01 +0300 Subject: [PATCH 04/20] Rename LD to CCLD The program links with CCLD, not raw LD. Escape confusion. --- Makefile | 7 ++++--- README.md | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 13ac831..7b548ae 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ BUILD_DIR := $(CURDIR)/build DEPSDIR := $(BUILD_DIR)/deps CC := gcc +CCLD := gcc LD := gcc CFLAGS:=-Wall -Wpedantic -Wno-unused-variable -I$(DEPSDIR)/include -Os LDFLAGS:=-L$(DEPSDIR)/lib -static @@ -18,7 +19,7 @@ else PREFIX := $(DESTDIR) endif -export CC LD CFLAGS LDFLAGS LIBNFNETLINK_CFLAGS LIBNFNETLINK_LIBS LIBMNL_CFLAGS LIBMNL_LIBS +export CC CCLD LD CFLAGS LDFLAGS LIBNFNETLINK_CFLAGS LIBNFNETLINK_LIBS LIBMNL_CFLAGS LIBMNL_LIBS APP:=$(BUILD_DIR)/youtubeUnblock @@ -63,8 +64,8 @@ $(LIBNETFILTER_QUEUE): $(LIBNFNETLINK) $(LIBMNL) $(MAKE) install -C deps/libnetfilter_queue $(APP): $(OBJS) $(LIBNETFILTER_QUEUE) $(LIBMNL) - @echo 'LD $(APP)' - @$(LD) $(OBJS) -o $(APP) -L$(DEPSDIR)/lib -lmnl -lnetfilter_queue + @echo 'CCLD $(APP)' + @$(CCLD) $(OBJS) -o $(APP) -L$(DEPSDIR)/lib -lmnl -lnetfilter_queue $(BUILD_DIR)/%.o: %.c $(LIBNETFILTER_QUEUE) $(LIBMNL) @echo 'CC $@' diff --git a/README.md b/README.md index 01a662b..e2e1bd7 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.serv Also DNS over HTTPS (DOH) is preferred for additional anonimity. ## OpenWRT case -The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-gcc AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. +The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc CCLD=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-ld AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. Now let's talk about a router configuration. I installed a normal iptables user-space app: `xtables-legacy iptables-zz-legacy` and kernel/iptables nfqueue extensions: `iptables-mod-nfqueue kmod-ipt-nfqueue` and add `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass` rule. From 13e78bde6d25556dbfa5f2ae09774f437cb4f4e8 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Wed, 31 Jul 2024 13:51:51 +0300 Subject: [PATCH 05/20] (feature) Solves #7 --- Makefile | 2 +- README.md | 2 +- youtubeUnblock.service | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 988ffc9..13ac831 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ install: all uninstall: rm $(PREFIX)/bin/youtubeUnblock rm $(PREFIX)/lib/systemd/system/youtubeUnblock.service - systemctl disable youtubeUnblock.service + -systemctl disable youtubeUnblock.service clean: rm -rf $(BUILD_DIR) diff --git a/README.md b/README.md index 1896b90..69a50df 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Here iptables serves every tcp packet, destinating port 443 for this userspace p Run an application with `youtubeUnblock 537` where `537` stands for the queue-num (must be the same as in the iptables rule). -Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). +Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). Please, note that systemd will configure iptables automatically. If you have troubles with it, delete ExecStartPre and ExecStop from youtubeUnblock.service and configure iptables manually (may be a useful case for nftables). Also DNS over HTTPS (DOH) is preferred for additional anonimity. diff --git a/youtubeUnblock.service b/youtubeUnblock.service index 672dc6a..fba168d 100644 --- a/youtubeUnblock.service +++ b/youtubeUnblock.service @@ -5,7 +5,9 @@ Description=youtubeUnblock StandardError=journal StandardOutput=journal StandardInput=null +ExecStartPre=iptables -t mangle -A OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass ExecStart=$(PREFIX)/bin/youtubeUnblock 537 +ExecStop=iptables -t mangle -D OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass [Install] WantedBy=multi-user.target From c290c9060400cf3e82503fbf7e228416501959b5 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Thu, 1 Aug 2024 09:58:34 +0300 Subject: [PATCH 06/20] Default CCLD to CC, LD to ld --- Makefile | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7b548ae..62d8111 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ BUILD_DIR := $(CURDIR)/build DEPSDIR := $(BUILD_DIR)/deps CC := gcc -CCLD := gcc -LD := gcc +CCLD := $(CC) +LD := ld CFLAGS:=-Wall -Wpedantic -Wno-unused-variable -I$(DEPSDIR)/include -Os LDFLAGS:=-L$(DEPSDIR)/lib -static diff --git a/README.md b/README.md index e2e1bd7..65df3e6 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.serv Also DNS over HTTPS (DOH) is preferred for additional anonimity. ## OpenWRT case -The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc CCLD=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-ld AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. +The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-ld AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. Now let's talk about a router configuration. I installed a normal iptables user-space app: `xtables-legacy iptables-zz-legacy` and kernel/iptables nfqueue extensions: `iptables-mod-nfqueue kmod-ipt-nfqueue` and add `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass` rule. From 4f78c490f484bf8dfd335e6a9e30c2d940e4a964 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Thu, 1 Aug 2024 12:35:03 +0300 Subject: [PATCH 07/20] Add option to specify 2nd packet delay --- youtubeUnblock.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 5c258fa..734f556 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifndef NOUSE_GSO #define USE_GSO @@ -29,6 +30,7 @@ #endif #define RAWSOCKET_MARK 0xfc70 +#define SEG2_DELAY 100 static struct { uint32_t queue_num; @@ -564,7 +566,23 @@ nextMessage: return vrd; } +struct dps_t { + struct pkt_buff *pkt; + // Time for the packet in milliseconds + uint32_t timer; +}; +// Note that the thread will automatically release dps_t and pkt_buff +void *delay_packet_send(void *data) { + struct dps_t *dpdt = data; + struct pkt_buff *pkt = dpdt->pkt; + usleep(dpdt->timer * 1000); + send_raw_socket(pkt); + + pktb_free(pkt); + free(dpdt); + return NULL; +} static int process_packet(const struct packet_data packet) { char buf[MNL_SOCKET_BUFFER_SIZE]; @@ -644,11 +662,31 @@ static int process_packet(const struct packet_data packet) { goto fallback; } - if ((send_raw_socket(frag2) == -1) || (send_raw_socket(frag1) == -1)) { + int ret = send_raw_socket(frag2); + if (ret < 0) { + errno = ret; perror("raw frags send"); + pktb_free(frag1); + goto err; } - pktb_free(frag1); + +#ifdef SEG2_DELAY + struct dps_t *dpdt = malloc(sizeof(struct dps_t)); + dpdt->pkt = frag1; + dpdt->timer = SEG2_DELAY; + pthread_t thr; + pthread_create(&thr, NULL, delay_packet_send, dpdt); + pthread_detach(thr); +#else + ret = send_raw_socket(frag1); + if (ret < 0) { + free(frag1); + goto err; + } + free(frag1); +#endif +err: pktb_free(frag2); pktb_free(pktb); From 57c5b1f6aa1eb21ff6594cc41a1c89d6bf284b58 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Thu, 1 Aug 2024 18:45:10 +0300 Subject: [PATCH 08/20] Implement fake SNI --- raw_replacements.h | 6 +++++ youtubeUnblock.c | 61 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 raw_replacements.h diff --git a/raw_replacements.h b/raw_replacements.h new file mode 100644 index 0000000..c6f86b2 --- /dev/null +++ b/raw_replacements.h @@ -0,0 +1,6 @@ +#ifndef RAW_REPLACEMENTS_H +#define RAW_REPLACEMENTS_H + +const char fake_sni[] = "\276(\001\273\366\234|\335\213\222\023\330\200\030\001\366\350e\000\000\001\001\b\n}\355\267Hm/\217\347\026\003\001\004\316\001\000\004\312\003\003K+\272\314\340\306\374>dw%\f\223\346\225\270\270~\335\027\f\264\341H\267\357\303\216T\322[\371 \245\320\212V6\374\3706\232\0216B\325\273P\b\300>\0332>\362\323\033\322\301\204\022f8\223\214\000\"\023\001\023\003\023\002\300+\300/\314\251\314\250\300,\3000\300\n\300\t\300\023\300\024\000\234\000\235\000/\0005\001\000\004_\000\000\000\023\000\021\000\000\016www.google.com\000\027\000\000\377\001\000\001\000\000\n\000\016\000\f\000\035\000\027\000\030\000\031\001\000\001\001\000\v\000\002\001\000\000\020\000\v\000\t\bhttp/1.1\000\005\000\005\001\000\000\000\000\000\"\000\n\000\b\004\003\005\003\006\003\002\003\0003\000k\000i\000\035\000 \333C\212\234-\t\237#\202\\\231\311\022]\333\341t(\t\276U\373u\234\316J~,^|*Z\000\027\000A\004k\n\255\254\376X\226t\001;n~\033\034.\245\027\024\3762_\352$\374\346^f\fF,\201\275\263\336O\231\001\032\200\357dI\266y\031\323\311vR\232\004\r\366FT\004\335\326\356\256\230B\t\313\000*\000\000\000+\000\005\004\003\004\003\003\000\r\000\030\000\026\004\003\005\003\006\003\b\004\b\005\b\006\004\001\005\001\006\001\002\003\002\001\000-\000\002\001\001\000\034\000\002@\001\376\r\0029\000\000\001\000\003\344\000 \337\306\243\332Y\033\a\252\352\025\365Z\035\223\226\304\255\363\215G\356g\344%}7\217\033n\211^\201\002\017g\267\334\326OD}\336\341ZC\230\226'\225\313\357\211\\\242\273\030k\216\377U\315\206\2410\200\203\332Z\223\005\370\b\304\370f\017\200\023\241\223~?\270{\037b\312\001\270\227\366\356\352\002\314\351\006\237\241q\226\300\314\321o\247{\201\317\230}B\005T\3660\335\320\332r?S\217\tq\036\031\326I|\237]\311 c\f\024r\031\310W\373\257\314q)q\030\237\261\227\217Kd?\257'G\320\020\340\256ND\247\005\341\324\024OP>\370\350\270b\311wAj\t\311\213\365i\203\230x\207\354\245<\274\202\230c\v0Y\263\364\022\303a\200\022\031\314\271rl=\327\336\001\327\264\267\342\353\352=\354[u\224\260\257\034\004\232\023\226}\227\030e\221\"\350\207\027dId\324\305\362N:\035\307`\204\337\201;\221\320\266b\362hrH\345e\206\246%\006\020a4\3430\036\225\215\274\275\360Q&\271\237)\222uK\362\017o\220\226W\357\267#\357\v\023\354\213\2629\331\ad\005/~6k\000[\247\301\270\310qJ\004\303|m5\363\376Y\002\243}6\251x\024\331)GH\335\205rI\032\f\210\a\212\347]\271\030\347.\021\213\365\026\030\340/Ny\r\332\3577\3203\026iX}>\2507\327&XRXU!\017\270I\313\352\350^?\352Uss\017\266pF\222NI\245\307_\305#\361\352\243+-\266\317Q\036s\243\277\355{S&\023>\275\360\215\032V\237XOY\345u>\002\305\252T\354\035\327v{P\352M\233\366\221\270\377\251\261f+rF\201wL2W\266X\252\242X\2536I\337c\205uZ\254Fe\305h\t\371\376\216r\336Y\327h\347*\331\257-ZQ{(\336\226\206\017\037\036\021\341\027z\033\254\235\252\227\224\004?p\243\351\\\263\352\205\327#W\345\255\256\375\267bP\3047\363!*K\003t\212(\306\214P\215\3506j\025\375\213e\254s\000)\001\034\000\367\000\361\002\276W%\232?\326\223\277\211v\017\a\361\347\312N\226\024L\260v\210\271j\324[|\270\344\3773\321-\313b>~\310\253XIR\324)&;\033{g;)\344\255\226\370\347I\\y\020\324\360\211vC\310\226s\267|\273$\341\332\2045qh\245w\2255\214\316\030\255\301\326C\343\304=\245\231h`yd\000#s\002\370\374Z\0336\245\361\226\222\306\032k\2457\016h\314(R;\326T~EHH\352\307\023^\247\363\321`V\340\253Z\233\357\227I\373\337z\177\nv\261\252\371\017\226\223\345\005\315y4\b\236N0\2630\017\215c\305&L\260\346J\237\203Q(\335W\027|>\3553\275j\307?W5\3463kc\350\262C\361 \037w!\371}\214\"I\377|\331@a;\342\3566\312\272Z\327u7\204'\215YBLL\235\236\242\345\215\245T\211a\312\263\342\000! \221\202X$\302\317\203\246\207c{\231\330\264\324\\k\271\272\336\356\002|\261O\207\030+\367P\317\356"; + +#endif /*RAW_REPLACEMENTS_H*/ diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 734f556..35c7272 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -21,6 +21,8 @@ #include #include +#include "raw_replacements.h" + #ifndef NOUSE_GSO #define USE_GSO #endif @@ -30,7 +32,15 @@ #endif #define RAWSOCKET_MARK 0xfc70 + +#ifndef NO_SEG2_DELAY #define SEG2_DELAY 100 +#endif + +#ifndef NO_FAKE_SNI +#define FAKE_SNI +#endif + static struct { uint32_t queue_num; @@ -566,6 +576,40 @@ nextMessage: return vrd; } +static struct pkt_buff *gen_fake_sni(const struct iphdr *iph, const struct tcphdr *tcph) { + int ip_len = iph->ihl * 4; + int tcp_len = tcph->doff * 4; + + size_t pkt_size = ip_len + sizeof(fake_sni); + struct pkt_buff *pkt = pktb_alloc(AF_INET, NULL, 0, pkt_size); + if (pkt == NULL) return NULL; + + pktb_mangle(pkt, 0, 0, 0, (const char *)iph, ip_len); + pktb_mangle(pkt, ip_len, 0, 0, fake_sni, sizeof(fake_sni)); + + int ret = 0; + struct iphdr *niph = nfq_ip_get_hdr(pkt); + if (!niph) goto err; + + ret = nfq_ip_set_transport_header(pkt, niph); + if (ret < 0) goto err; + + struct tcphdr *ntcph = nfq_tcp_get_hdr(pkt); + if (!ntcph) goto err; + + niph->tot_len = htons(pkt_size); + ntcph->th_dport = tcph->th_dport; + ntcph->th_sport = tcph->th_sport; + nfq_ip_set_checksum(niph); + nfq_tcp_compute_checksum_ipv4(ntcph, niph); + + return pkt; +err: + fprintf(stderr, "Error in gen fake sni\n"); + pktb_free(pkt); + return NULL; + +} struct dps_t { struct pkt_buff *pkt; // Time for the packet in milliseconds @@ -647,6 +691,19 @@ static int process_packet(const struct packet_data packet) { #ifdef USE_TCP_SEGMENTATION + struct pkt_buff *fake_sni = gen_fake_sni(ip_header, tcph); + if (fake_sni == NULL) goto fallback; + + int ret = 0; +#ifdef FAKE_SNI + ret = send_raw_socket(fake_sni); +#endif + if (ret < 0) { + perror("send fake sni\n"); + pktb_free(fake_sni); + goto fallback; + } + size_t ipd_offset = vrd.sni_offset; size_t mid_offset = ipd_offset + vrd.sni_len / 2; @@ -659,10 +716,11 @@ static int process_packet(const struct packet_data packet) { if (tcp4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { perror("tcp4_frag"); pktb_free(pktb); + pktb_free(fake_sni); goto fallback; } - int ret = send_raw_socket(frag2); + ret = send_raw_socket(frag2); if (ret < 0) { errno = ret; perror("raw frags send"); @@ -689,6 +747,7 @@ static int process_packet(const struct packet_data packet) { err: pktb_free(frag2); pktb_free(pktb); + pktb_free(fake_sni); #else // TODO: Implement compute of tcp checksum From 8808c49fbdac5c7f8e1340dc07969541d2e3e3c1 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Thu, 1 Aug 2024 20:54:02 +0300 Subject: [PATCH 09/20] Cleanup code, update readme, fix gen_fake_sni --- README.md | 11 +++++++---- youtubeUnblock.c | 51 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 69a50df..fd094a9 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,13 @@ Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.serv Also DNS over HTTPS (DOH) is preferred for additional anonimity. +## Troubleshooting +If you have any troubles with youtubeUnblock, here are some options to tune. If them don't work in your case, please, open an issue. You can pass these options in make CFLAGS (`make CFLAGS=...`) or edit CFLAGS variable in Makefile. +Available flags: +- -DUSE_SEG2_DELAY This flag forces youtubeUnblock to wait little bit before send the 2nd part of the split packet. You can tune the amount of time in `#define SEG2_DELAY 100` where 100 stands for milliseconds. +- -DNO_FAKE_SNI This flag forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. +- -DNOUSE_GSO This flag disables fix for Google Chrome fat ClientHello. The GSO is well tested now, so this flag probably won't fix anything. + ## OpenWRT case The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-gcc AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. @@ -37,8 +44,4 @@ Now let's talk about a router configuration. I installed a normal iptables user- Next step is to daemonize the application in openwrt. Copy youtubeUnblock.owrt to /etc/init.d/youtubeUnblock and put the program into /usr/bin/. (Don't forget to `chmod +x` both). Now run `/etc/init.d/youtubeUnblock start`. You can alo run `/etc/init.d/youtubeUnblock enable` to force OpenWRT autostart the program on boot. - -## Further development -Please note that the application needs in further development. Some googlevideo servers may still be unabailable, some may drop out hello packets on Firefox while some may do so on Chrome. If you got in trouble try to disable GSO (Pass -DNOUSE_GSO as CC_FLAGS). Also you may set the program to use IP fragmentation instead of TCP (-DUSE_IP_FRAGMENTATION). - **If you have any questions/suggestions/problems feel free to open an issue.** diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 35c7272..d899c07 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -33,7 +33,7 @@ #define RAWSOCKET_MARK 0xfc70 -#ifndef NO_SEG2_DELAY +#ifdef USE_SEG2_DELAY #define SEG2_DELAY 100 #endif @@ -589,15 +589,26 @@ static struct pkt_buff *gen_fake_sni(const struct iphdr *iph, const struct tcphd int ret = 0; struct iphdr *niph = nfq_ip_get_hdr(pkt); - if (!niph) goto err; + if (!niph) { + perror("gen_fake_sni: ip header is null"); + goto err; + } + + niph->protocol = IPPROTO_TCP; + niph->tot_len = htons(pkt_size); ret = nfq_ip_set_transport_header(pkt, niph); - if (ret < 0) goto err; + if (ret < 0) { + perror("gen_fake_sni: set transport header"); + goto err; + } struct tcphdr *ntcph = nfq_tcp_get_hdr(pkt); - if (!ntcph) goto err; + if (!ntcph) { + perror("gen_fake_sni: nfq_tcp_get_hdr"); + goto err; + } - niph->tot_len = htons(pkt_size); ntcph->th_dport = tcph->th_dport; ntcph->th_sport = tcph->th_sport; nfq_ip_set_checksum(niph); @@ -605,7 +616,6 @@ static struct pkt_buff *gen_fake_sni(const struct iphdr *iph, const struct tcphd return pkt; err: - fprintf(stderr, "Error in gen fake sni\n"); pktb_free(pkt); return NULL; @@ -712,6 +722,12 @@ static int process_packet(const struct packet_data packet) { packet.payload, packet.payload_len, 0); + + if (pktb == NULL) { + perror("pktb_alloc of payload"); + pktb_free(fake_sni); + goto fallback; + } if (tcp4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { perror("tcp4_frag"); @@ -723,11 +739,10 @@ static int process_packet(const struct packet_data packet) { ret = send_raw_socket(frag2); if (ret < 0) { errno = ret; - perror("raw frags send"); + perror("raw frags send: frag2"); pktb_free(frag1); goto err; } - #ifdef SEG2_DELAY struct dps_t *dpdt = malloc(sizeof(struct dps_t)); @@ -739,10 +754,12 @@ static int process_packet(const struct packet_data packet) { #else ret = send_raw_socket(frag1); if (ret < 0) { - free(frag1); + errno = ret; + perror("raw frags send: frag1"); + pktb_free(frag1); goto err; } - free(frag1); + pktb_free(frag1); #endif err: pktb_free(frag2); @@ -851,9 +868,17 @@ int main(int argc, const char *argv[]) } #ifdef USE_TCP_SEGMENTATION - printf("Using TCP segmentation!\n"); + printf("Using TCP segmentation\n"); #else - printf("Using IP fragmentation!\n"); + printf("Using IP fragmentation\n"); +#endif + +#ifdef SEG2_DELAY + printf("Some outgoing googlevideo request segments will be delayed for %d ms as of SEG2_DELAY define\n", SEG2_DELAY); +#endif + +#ifdef FAKE_SNI + printf("Fake SNI will be sent before each googlevideo request\n"); #endif if (open_socket()) { @@ -889,7 +914,7 @@ int main(int argc, const char *argv[]) nfq_nlmsg_cfg_put_params(nlh, NFQNL_COPY_PACKET, 0xffff); #ifdef USE_GSO - printf("GSO is enabled!\n"); + printf("GSO is enabled\n"); mnl_attr_put_u32(nlh, NFQA_CFG_FLAGS, htonl(NFQA_CFG_F_GSO)); mnl_attr_put_u32(nlh, NFQA_CFG_MASK, htonl(NFQA_CFG_F_GSO)); From a96d6213b1108785acec960f054f9efde5d77fdc Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Fri, 2 Aug 2024 10:19:04 +0300 Subject: [PATCH 10/20] Temporary fix #17 This commit will entirely disable Fake SNI if flag -DNO_FAKE_SNI is set --- Makefile | 2 +- README.md | 2 +- youtubeUnblock.c | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 13ac831..43331c9 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ DEPSDIR := $(BUILD_DIR)/deps CC := gcc LD := gcc -CFLAGS:=-Wall -Wpedantic -Wno-unused-variable -I$(DEPSDIR)/include -Os +CFLAGS:=-Wall -Wpedantic -Wno-unused-variable -I$(DEPSDIR)/include -Os LDFLAGS:=-L$(DEPSDIR)/lib -static LIBNFNETLINK_CFLAGS := -I$(DEPSDIR)/include diff --git a/README.md b/README.md index fd094a9..49d9637 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Also DNS over HTTPS (DOH) is preferred for additional anonimity. If you have any troubles with youtubeUnblock, here are some options to tune. If them don't work in your case, please, open an issue. You can pass these options in make CFLAGS (`make CFLAGS=...`) or edit CFLAGS variable in Makefile. Available flags: - -DUSE_SEG2_DELAY This flag forces youtubeUnblock to wait little bit before send the 2nd part of the split packet. You can tune the amount of time in `#define SEG2_DELAY 100` where 100 stands for milliseconds. -- -DNO_FAKE_SNI This flag forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. +- -DNO_FAKE_SNI This flag disables -DFAKE_SNI which forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. Use this flag if you got gen_fake_sni error. Track this issue in https://github.com/Waujito/youtubeUnblock/issues/17 - -DNOUSE_GSO This flag disables fix for Google Chrome fat ClientHello. The GSO is well tested now, so this flag probably won't fix anything. ## OpenWRT case diff --git a/youtubeUnblock.c b/youtubeUnblock.c index d899c07..d440795 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -701,18 +701,18 @@ static int process_packet(const struct packet_data packet) { #ifdef USE_TCP_SEGMENTATION + int ret = 0; +#ifdef FAKE_SNI struct pkt_buff *fake_sni = gen_fake_sni(ip_header, tcph); if (fake_sni == NULL) goto fallback; - int ret = 0; -#ifdef FAKE_SNI ret = send_raw_socket(fake_sni); -#endif if (ret < 0) { perror("send fake sni\n"); pktb_free(fake_sni); goto fallback; } +#endif size_t ipd_offset = vrd.sni_offset; size_t mid_offset = ipd_offset + vrd.sni_len / 2; @@ -725,14 +725,19 @@ static int process_packet(const struct packet_data packet) { if (pktb == NULL) { perror("pktb_alloc of payload"); +#ifdef FAKE_SNI pktb_free(fake_sni); +#endif goto fallback; } if (tcp4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { perror("tcp4_frag"); pktb_free(pktb); +#ifdef FAKE_SNI pktb_free(fake_sni); +#endif + goto fallback; } @@ -764,7 +769,9 @@ static int process_packet(const struct packet_data packet) { err: pktb_free(frag2); pktb_free(pktb); +#ifdef FAKE_SNI pktb_free(fake_sni); +#endif #else // TODO: Implement compute of tcp checksum From 72a7c21b17144c65c4a2d232119a34f2c3e9826d Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Fri, 2 Aug 2024 13:43:59 +0300 Subject: [PATCH 11/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1fc338..1585b39 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # youtubeUnblock Bypasses Googlevideo detection systems that relies on SNI. The package is for Linux only. -For Windows use [GoodbyeDPI from ValdikSS](https://github.com/ValdikSS/GoodbyeDPI) (Users points out that working options for it are `goodbyedpi.exe -6 --frag-by-sni`.) The same behavior is also implemented in [zapret package for linux](https://github.com/bol-van/zapret). +For Windows use [GoodbyeDPI from ValdikSS](https://github.com/ValdikSS/GoodbyeDPI) (you can find how to use it for YouTube [here](https://github.com/ValdikSS/GoodbyeDPI/issues/378)) The same behavior is also implemented in [zapret package for linux](https://github.com/bol-van/zapret). ## How it works: Lets look from the DPIses side of view: All they have is ip and tcp information, higher-level data is encrypted. So from the IP header only IP address might be helpful for them. In tcp here is basically nothing. So they may handle IP addresses and process it. What's wrong? Google servers are on the way: It is very hard to handle all that infrastracture. One server may host multiple websites and it is very bad if them block, say Google Search trying to block googlevideo. But even if googlevideo servers have their own ip for only googlevideo purposes, here is a problem about how large is Google infrastracture and how much servers are here. The DPIs can't even parse normally all the servers, because each video may live on it's cache server. So what's else? Let's take a look at a TLS level. All information here is encrypted. All... Except hello messages! They are used to initialize handshake connections and hold tons of helpful information. If we talk about TLS v1.3, it is optimized to transfer as less information as possible unencrypted. But here is only one thing that may point us which domain the user wants to connect, the SNI extension. It transfers all domain names unencrypted. Exactly what we need! And DPIs may use this thing to detect google video connections and slow down them (In fact they are corrupting a tcp connection with bad packets). From 709fc4c3be67d92ce154f26a21deca8e2f7c81d5 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Fri, 2 Aug 2024 15:43:03 +0300 Subject: [PATCH 12/20] Fix #17 --- youtubeUnblock.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/youtubeUnblock.c b/youtubeUnblock.c index d440795..6ef171c 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -577,14 +577,20 @@ nextMessage: } static struct pkt_buff *gen_fake_sni(const struct iphdr *iph, const struct tcphdr *tcph) { + uint8_t sniph_buf[60]; int ip_len = iph->ihl * 4; - int tcp_len = tcph->doff * 4; - size_t pkt_size = ip_len + sizeof(fake_sni); + + memcpy(sniph_buf, iph, ip_len); + struct iphdr *sniph = (struct iphdr *)sniph_buf; + + sniph->protocol = IPPROTO_TCP; + sniph->tot_len = htons(pkt_size); + struct pkt_buff *pkt = pktb_alloc(AF_INET, NULL, 0, pkt_size); if (pkt == NULL) return NULL; - pktb_mangle(pkt, 0, 0, 0, (const char *)iph, ip_len); + pktb_mangle(pkt, 0, 0, 0, (char *)sniph_buf, ip_len); pktb_mangle(pkt, ip_len, 0, 0, fake_sni, sizeof(fake_sni)); int ret = 0; @@ -594,9 +600,6 @@ static struct pkt_buff *gen_fake_sni(const struct iphdr *iph, const struct tcphd goto err; } - niph->protocol = IPPROTO_TCP; - niph->tot_len = htons(pkt_size); - ret = nfq_ip_set_transport_header(pkt, niph); if (ret < 0) { perror("gen_fake_sni: set transport header"); From 571692b6bd3359663f03a8811652d8fbf81586ff Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Fri, 2 Aug 2024 23:00:08 +0300 Subject: [PATCH 13/20] Fix #17. Update troubleshooting documentation, update packet marks. Update for marks may be related to #15. But is primarily used to avoid random EPERMs. --- README.md | 10 +++++++++- youtubeUnblock.c | 26 ++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1585b39..e5a8211 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,17 @@ Also DNS over HTTPS (DOH) is preferred for additional anonimity. If you have any troubles with youtubeUnblock, here are some options to tune. If them don't work in your case, please, open an issue. You can pass these options in make CFLAGS (`make CFLAGS=...`) or edit CFLAGS variable in Makefile. Available flags: - -DUSE_SEG2_DELAY This flag forces youtubeUnblock to wait little bit before send the 2nd part of the split packet. You can tune the amount of time in `#define SEG2_DELAY 100` where 100 stands for milliseconds. -- -DNO_FAKE_SNI This flag disables -DFAKE_SNI which forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. Use this flag if you got gen_fake_sni error. Track this issue in https://github.com/Waujito/youtubeUnblock/issues/17 +- -DNO_FAKE_SNI This flag disables -DFAKE_SNI which forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. This flag may be related to some Operation not permitted error messages, so befor open an issue refer to FAQ for EPERMS. - -DNOUSE_GSO This flag disables fix for Google Chrome fat ClientHello. The GSO is well tested now, so this flag probably won't fix anything. +### FAQ for EPERMS (Operation not permitted) +EPERM may occur in a lot of places but generally here are two: mnl_cb_run and when sending the packet via rawsocket (raw_frags_send and send fake sni). +- mnl_cb_run Operation not permitted indicates that another instance of youtubeUnblock is running on the specified queue-num. +- rawsocket Operation not permitted indicates that the packet is being dropped by nefilter rules. In fact this is a hint from the kernel that something wrong is going on and we should check the firewall rules. Before dive into the problem let's make it clean how the mangled packets are being sent. Nefilter queue provides us with the ability to mangle the packet on fly but that is not suitable for this program because we need to split the packet to at least two independent packets. So we are using [linux raw sockets](https://man7.org/linux/man-pages/man7/raw.7.html) which allows us to send any ipv4 packet. **The packet goes from the OUTPUT chain even when NFQUEUE is set up on FORWARD (suitable for OpenWRT).** So we need to escape packet rejects here. + * raw_frags_send EPERM: just make sure outgoing traffic is allowed (RELATED,ESTABLISHED should work, if not, go to step 3) + * send fake sni EPERM: Fake SNI is out-of-state thing and will likely corrupt the connection (the behavior is expected). conntrack considers it as an invalid packet. By default OpenWRT set up to drop outgoing packets like this one. You may delete nftables/iptables rule that drops packets with invalid conntrack state, but I don't recommend to do this. The step 3 is better solution. + * Step 3, ultimate solution. Use mark (don't confuse with connmark). The youtubeUnblock uses mark internally to avoid infinity packet loops (when the packet is sent by youtubeUnblock but on next step handled by itself). Currently it uses mark (1 << 15) = 32768. You should put iptables/nftables that ultimately accepts such marks at the very start of the filter OUTPUT chain: `iptables -I OUTPUT -m mark --mark 32768/32768 -j ACCEPT` or `nft insert rule inet fw4 output mark and 0x8000 == 0x8000 counter accept`. + ## OpenWRT case The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-ld AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 6ef171c..88a4a91 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -31,7 +31,7 @@ #define USE_TCP_SEGMENTATION #endif -#define RAWSOCKET_MARK 0xfc70 +#define RAWSOCKET_MARK (1 << 15) #ifdef USE_SEG2_DELAY #define SEG2_DELAY 100 @@ -251,18 +251,27 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, struct iphdr *hdr = nfq_ip_get_hdr(pktb); size_t hdr_len = hdr->ihl * 4; - if (hdr == NULL) {errno = EINVAL; return -1;} + if (hdr == NULL) { + perror("tcp4_frag: nfq_ip_get_hdr is null\n"); + errno = EINVAL; + return -1; + } if (hdr->protocol != IPPROTO_TCP || !(ntohs(hdr->frag_off) & IP_DF)) { + perror("tcp4_frag: proto is tcp or request is fragmented"); errno = EINVAL; return -1; } - if (nfq_ip_set_transport_header(pktb, hdr)) + if (nfq_ip_set_transport_header(pktb, hdr)) { + perror("tcp4_frag: ip_set_transport_header"); return -1; + } + struct tcphdr *tcph = nfq_tcp_get_hdr(pktb); size_t tcph_len = tcph->doff * 4; if (tcph == NULL) { + perror("tcp4_frag: tcph is NULL"); errno = EINVAL; return -1; } @@ -270,7 +279,8 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, uint8_t *payload = nfq_tcp_get_payload(tcph, pktb); size_t plen = nfq_tcp_get_payload_len(tcph, pktb); - if (hdr == NULL || payload == NULL || plen <= payload_offset) { + if (payload == NULL || plen <= payload_offset) { + perror("tcp4_frag: payload is too small or NULL"); errno = EINVAL; return -1; } @@ -280,6 +290,10 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, size_t s2_plen = plen - payload_offset; size_t s2_dlen = s2_plen + hdr_len + tcph_len; + if (s1_dlen > MNL_SOCKET_BUFFER_SIZE || s2_dlen > MNL_SOCKET_BUFFER_SIZE) { + errno = ENOMEM; + return -1; + } memcpy(buff1, hdr, hdr_len); memcpy(buff2, hdr, hdr_len); @@ -711,7 +725,7 @@ static int process_packet(const struct packet_data packet) { ret = send_raw_socket(fake_sni); if (ret < 0) { - perror("send fake sni\n"); + perror("send fake sni"); pktb_free(fake_sni); goto fallback; } @@ -859,7 +873,7 @@ static int queue_cb(const struct nlmsghdr *nlh, void *data) { if (attr[NFQA_MARK] != NULL) { // Skip packets sent by rawsocket to escape infinity loop. - if (ntohl(mnl_attr_get_u32(attr[NFQA_MARK])) == + if ((ntohl(mnl_attr_get_u32(attr[NFQA_MARK])) & RAWSOCKET_MARK) == RAWSOCKET_MARK) { return fallback_accept_packet(packet.id); } From 7dd45e43146a2ae0b308a1a3c28c0ea0a1293732 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Fri, 2 Aug 2024 23:34:48 +0300 Subject: [PATCH 14/20] Update README.md --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e5a8211..f20d84c 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,11 @@ Before compilation make sure `gcc`, `make`, `autoconf`, `automake`, `pkg-config` Compile with `make`. Install with `make install`. The package include libnetfilter_queue, libnfnetlink and libmnl as static dependencies. The package requires linux-headers and kernel built with netfilter nfqueue support. You should also configure iptables for this to start working: -```iptables -A OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass``` +```iptables -A OUTPUT -p tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass``` (or do ```nft add rule ip mangle OUTPUT tcp dport 443 counter queue num 537 bypass``` for nftables.) Here iptables serves every tcp packet, destinating port 443 for this userspace packet analyzer (via netfilter kernel module) queue-num may be any number from 0 to 65565. --queue-bypass allows traffic to pass if the application is down. +Also tips to explicitly accept all important outgoing raw packets from youtubeUnblock from [Troubleshooting EPERMS](https://github.com/Waujito/youtubeUnblock?tab=readme-ov-file#troubleshooting-eperms-operation-not-permitted) may be useful to avoid issues. + Run an application with `youtubeUnblock 537` where `537` stands for the queue-num (must be the same as in the iptables rule). Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). Please, note that systemd will configure iptables automatically. If you have troubles with it, delete ExecStartPre and ExecStop from youtubeUnblock.service and configure iptables manually (may be a useful case for nftables). @@ -37,7 +39,7 @@ Available flags: - -DNO_FAKE_SNI This flag disables -DFAKE_SNI which forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. This flag may be related to some Operation not permitted error messages, so befor open an issue refer to FAQ for EPERMS. - -DNOUSE_GSO This flag disables fix for Google Chrome fat ClientHello. The GSO is well tested now, so this flag probably won't fix anything. -### FAQ for EPERMS (Operation not permitted) +### Troubleshooting EPERMS (Operation not permitted) EPERM may occur in a lot of places but generally here are two: mnl_cb_run and when sending the packet via rawsocket (raw_frags_send and send fake sni). - mnl_cb_run Operation not permitted indicates that another instance of youtubeUnblock is running on the specified queue-num. - rawsocket Operation not permitted indicates that the packet is being dropped by nefilter rules. In fact this is a hint from the kernel that something wrong is going on and we should check the firewall rules. Before dive into the problem let's make it clean how the mangled packets are being sent. Nefilter queue provides us with the ability to mangle the packet on fly but that is not suitable for this program because we need to split the packet to at least two independent packets. So we are using [linux raw sockets](https://man7.org/linux/man-pages/man7/raw.7.html) which allows us to send any ipv4 packet. **The packet goes from the OUTPUT chain even when NFQUEUE is set up on FORWARD (suitable for OpenWRT).** So we need to escape packet rejects here. @@ -48,8 +50,13 @@ EPERM may occur in a lot of places but generally here are two: mnl_cb_run and wh ## OpenWRT case The package is also compatible with routers. The router should be running by free opensource linux-based system such as [OpenWRT](https://openwrt.org/). You should cross-compile it under your host machine. Be ready for compilation errors and a lot of googling about it. It is not such a trivial process! You can get crosscompilation toolsuite compatible with your router from OpenWRT repositories. For example, I have ramips/mt76x8 based router so for me the toolsuite is on https://downloads.openwrt.org/releases/23.05.3/targets/ramips/mt76x8/ and called `openwrt-toolchain-23.05.3-ramips-mt76x8_gcc-12.3.0_musl.Linux-x86_64.tar.xz`. You can find out more about your router model on it's openwrt page. When you download the toolsuite, untar it somewhere. Now we are ready for compilation. My cross gcc asked me to create a staging dir for it and pass it as an environment variable. Also you should notice toolsuite packages and replace my make command with yours. ```STAGING_DIR=temp make CC=/usr/bin/mipsel-openwrt-linux-gcc LD=/usr/bin/mipsel-openwrt-linux-ld AR=/usr/bin/mipsel-openwrt-linux-ar OBJDUMP=/usr/bin/mipsel-openwrt-linux-objdump NM=/usr/bin/mipsel-openwrt-linux-nm STRIP=/usr/bin/mipsel-openwrt-linux-strip CROSS_COMPILE_PLATFORM=mipsel-buildroot-linux-gnu```. Take a look at `CROSS_COMPILE_PLATFORM` It is required by autotools but I think it is not necessary. Anyways I put `mipsel-buildroot-linux-gnu` in here. For your model may be an [automake cross-compile manual](https://www.gnu.org/software/automake/manual/html_node/Cross_002dCompilation.html) will be helpful. When compilation is done, the binary file will be in build directory. Copy it to your router. Note that an ssh access is likely to be required to proceed. sshfs don't work on my model so I injected the application to the router via Software Upload Package page. It has given me an error, but also a `/tmp/upload.ipk` file which I copied in root directory, `chmod +x`-ed and run. -Now let's talk about a router configuration. I installed a normal iptables user-space app: `xtables-legacy iptables-zz-legacy` and kernel/iptables nfqueue extensions: `iptables-mod-nfqueue kmod-ipt-nfqueue` and add `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass` rule. +Now let's talk about a router configuration. I installed a normal iptables user-space app: `xtables-legacy iptables-zz-legacy` and kernel/iptables nfqueue extensions: `iptables-mod-nfqueue kmod-ipt-nfqueue` and add `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass` rule. + +If you prefer nftables this should work: `nft add rule inet fw4 mangle_forward tcp dport 443 counter queue num 537 bypass`. Note that kmod-nft-queue should be installed. Next step is to daemonize the application in openwrt. Copy youtubeUnblock.owrt to /etc/init.d/youtubeUnblock and put the program into /usr/bin/. (Don't forget to `chmod +x` both). Now run `/etc/init.d/youtubeUnblock start`. You can alo run `/etc/init.d/youtubeUnblock enable` to force OpenWRT autostart the program on boot. +## Performance +If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. + **If you have any questions/suggestions/problems feel free to open an issue.** From a333b9bfe67fba401d98d3b4bd3c9e3059df6228 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sat, 3 Aug 2024 00:20:19 +0300 Subject: [PATCH 15/20] Update README.md for OpenWRT, enhance OpenWRT build system. --- README.md | 10 ++++++---- owrt/537-youtubeUnblock.nft | 5 +++++ youtubeUnblock.owrt => owrt/youtubeUnblock.owrt | 0 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 owrt/537-youtubeUnblock.nft rename youtubeUnblock.owrt => owrt/youtubeUnblock.owrt (100%) diff --git a/README.md b/README.md index f20d84c..341e8b1 100644 --- a/README.md +++ b/README.md @@ -52,11 +52,13 @@ The package is also compatible with routers. The router should be running by fre Now let's talk about a router configuration. I installed a normal iptables user-space app: `xtables-legacy iptables-zz-legacy` and kernel/iptables nfqueue extensions: `iptables-mod-nfqueue kmod-ipt-nfqueue` and add `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -j NFQUEUE --queue-num 537 --queue-bypass` rule. -If you prefer nftables this should work: `nft add rule inet fw4 mangle_forward tcp dport 443 counter queue num 537 bypass`. Note that kmod-nft-queue should be installed. +If you prefer nftables, this should work: `nft add rule inet fw4 mangle_forward tcp dport 443 counter queue num 537 bypass`. Note that kmod-nft-queue should be installed. -Next step is to daemonize the application in openwrt. Copy youtubeUnblock.owrt to /etc/init.d/youtubeUnblock and put the program into /usr/bin/. (Don't forget to `chmod +x` both). Now run `/etc/init.d/youtubeUnblock start`. You can alo run `/etc/init.d/youtubeUnblock enable` to force OpenWRT autostart the program on boot. +Also you can copy `owrt/537-youtubeUnblock.nft` to `/usr/share/nftables.d/ruleset-post/537-youtubeUnblock.nft` and run `/etc/init.d/firewall reload`. This will reload the nftables ruleset and automatically link 537-youtubeUnblock.nft with it. + +Next step is to daemonize the application in openwrt. Copy `owrt/youtubeUnblock.owrt` to `/etc/init.d/youtubeUnblock` and put the program into /usr/bin/. (Don't forget to `chmod +x` both). Now run `/etc/init.d/youtubeUnblock start`. You can alo run `/etc/init.d/youtubeUnblock enable` to force OpenWRT autostart the program on boot, but I don't recommend this since if the packet has bug you may lose access to the router (I think you will be able to reset it with reset settings tricks documented for your router). ## Performance -If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. +If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets from the connection. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. -**If you have any questions/suggestions/problems feel free to open an issue.** +## If you have any questions/suggestions/problems feel free to open an issue. diff --git a/owrt/537-youtubeUnblock.nft b/owrt/537-youtubeUnblock.nft new file mode 100644 index 0000000..e774b46 --- /dev/null +++ b/owrt/537-youtubeUnblock.nft @@ -0,0 +1,5 @@ +#!/usr/sbin/nft -f +# This file + +add rule inet fw4 mangle_forward tcp dport 443 ct packets < 20 counter queue num 537 bypass +insert rule inet fw4 output mark and 0x8000 == 0x8000 counter accept diff --git a/youtubeUnblock.owrt b/owrt/youtubeUnblock.owrt similarity index 100% rename from youtubeUnblock.owrt rename to owrt/youtubeUnblock.owrt From ac9f49f937b38d331156ca36537b6952572c531c Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sat, 3 Aug 2024 01:27:23 +0300 Subject: [PATCH 16/20] Be more verbose I have tired to always enable DEBUG and I dont see the reason why verbosity should be disabled by default --- README.md | 2 ++ youtubeUnblock.c | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 341e8b1..2baa680 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Run an application with `youtubeUnblock 537` where `537` stands for the queue-nu Systemd daemon is also available. Do `systemctl enable --now youtubeUnblock.service` after installation (uses queue-num `537`). Please, note that systemd will configure iptables automatically. If you have troubles with it, delete ExecStartPre and ExecStop from youtubeUnblock.service and configure iptables manually (may be a useful case for nftables). +If you don't want youtubeUnblock to log on each googlevideo request pass -DSILENT as CFLAGS. + Also DNS over HTTPS (DOH) is preferred for additional anonimity. ## Troubleshooting diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 88a4a91..544f38c 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -41,6 +41,9 @@ #define FAKE_SNI #endif +#ifndef SILENT +#define DEBUG +#endif static struct { uint32_t queue_num; @@ -721,7 +724,10 @@ static int process_packet(const struct packet_data packet) { int ret = 0; #ifdef FAKE_SNI struct pkt_buff *fake_sni = gen_fake_sni(ip_header, tcph); - if (fake_sni == NULL) goto fallback; + if (fake_sni == NULL) { + perror("gen_fake_sni"); + goto fallback; + } ret = send_raw_socket(fake_sni); if (ret < 0) { From 1eb95f50f572f408e341566d4ca6462d8dbab268 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sat, 3 Aug 2024 01:55:19 +0300 Subject: [PATCH 17/20] Kyber on Chromium --- README.md | 2 ++ owrt/537-youtubeUnblock.nft | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2baa680..14a41c9 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ Available flags: - -DNO_FAKE_SNI This flag disables -DFAKE_SNI which forces youtubeUnblock to send at least three packets instead of one with TLS ClientHello: Fake ClientHello, 1st part of original ClientHello, 2nd part of original ClientHello. This flag may be related to some Operation not permitted error messages, so befor open an issue refer to FAQ for EPERMS. - -DNOUSE_GSO This flag disables fix for Google Chrome fat ClientHello. The GSO is well tested now, so this flag probably won't fix anything. +If you are on Chromium you may have to disable kyber (the feature that makes the TLS ClientHello very fat). I've got the problem with it on router, so to escape possibly errors it is better to just disable it: in chrome://flags search for kyber and switch it to disabled state. + ### Troubleshooting EPERMS (Operation not permitted) EPERM may occur in a lot of places but generally here are two: mnl_cb_run and when sending the packet via rawsocket (raw_frags_send and send fake sni). - mnl_cb_run Operation not permitted indicates that another instance of youtubeUnblock is running on the specified queue-num. diff --git a/owrt/537-youtubeUnblock.nft b/owrt/537-youtubeUnblock.nft index e774b46..34f7c84 100644 --- a/owrt/537-youtubeUnblock.nft +++ b/owrt/537-youtubeUnblock.nft @@ -1,5 +1,5 @@ #!/usr/sbin/nft -f # This file -add rule inet fw4 mangle_forward tcp dport 443 ct packets < 20 counter queue num 537 bypass +insert rule inet fw4 mangle_forward tcp dport 443 ct packets < 20 counter queue num 537 bypass insert rule inet fw4 output mark and 0x8000 == 0x8000 counter accept From ba4dce1df1edd27b6185b225e5f3cc7c50d9a498 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sat, 3 Aug 2024 09:38:22 +0300 Subject: [PATCH 18/20] Update performance settings for openwrt Add iptables connbytes counter thanks to this comment https://github.com/Waujito/youtubeUnblock/issues/17#issuecomment-2266359878 --- README.md | 4 +++- owrt/537-youtubeUnblock.nft | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 14a41c9..e66931a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ Also you can copy `owrt/537-youtubeUnblock.nft` to `/usr/share/nftables.d/rulese Next step is to daemonize the application in openwrt. Copy `owrt/youtubeUnblock.owrt` to `/etc/init.d/youtubeUnblock` and put the program into /usr/bin/. (Don't forget to `chmod +x` both). Now run `/etc/init.d/youtubeUnblock start`. You can alo run `/etc/init.d/youtubeUnblock enable` to force OpenWRT autostart the program on boot, but I don't recommend this since if the packet has bug you may lose access to the router (I think you will be able to reset it with reset settings tricks documented for your router). ## Performance -If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets from the connection. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. +If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets from the connection. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct original "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. + +The same behavior is also possible in iptables: `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -m connbytes --connbytes-dir original --connbytes-mode packets --connbytes 0:19 -j NFQUEUE --queue-num 537 --queue-bypass` ## If you have any questions/suggestions/problems feel free to open an issue. diff --git a/owrt/537-youtubeUnblock.nft b/owrt/537-youtubeUnblock.nft index 34f7c84..c70a2cd 100644 --- a/owrt/537-youtubeUnblock.nft +++ b/owrt/537-youtubeUnblock.nft @@ -1,5 +1,5 @@ #!/usr/sbin/nft -f # This file -insert rule inet fw4 mangle_forward tcp dport 443 ct packets < 20 counter queue num 537 bypass +insert rule inet fw4 mangle_forward tcp dport 443 ct original packets < 20 counter queue num 537 bypass insert rule inet fw4 output mark and 0x8000 == 0x8000 counter accept From 4793f537deba1627d838d8bb2e278d1a4d22b7c9 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sat, 3 Aug 2024 10:26:10 +0300 Subject: [PATCH 19/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e66931a..dc1c077 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,6 @@ Next step is to daemonize the application in openwrt. Copy `owrt/youtubeUnblock. ## Performance If you have bad performance you can queue to youtubeUnblock only first, say, 20 packets from the connection. To do so, use nftables conntrack packets counter: `nft add rule inet fw4 mangle_forward tcp dport 443 ct original "packets < 20" counter queue num 537 bypass`. For my 1 CPU core device it worked pretty well. This works because we do care about only first packets with ClientHello. We don't need to process others. -The same behavior is also possible in iptables: `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -m connbytes --connbytes-dir original --connbytes-mode packets --connbytes 0:19 -j NFQUEUE --queue-num 537 --queue-bypass` +The same behavior is also possible in iptables: `iptables -t mangle -A FORWARD -p tcp -m tcp --dport 443 -m connbytes --connbytes-dir original --connbytes-mode packets --connbytes 0:19 -j NFQUEUE --queue-num 537 --queue-bypass`. (The package iptables-mod-conntrack-extra is required for connbytes on OpenWRT) ## If you have any questions/suggestions/problems feel free to open an issue. From be40bc91e71536cfc1b2ab0da04f3587904c8f11 Mon Sep 17 00:00:00 2001 From: Vadim Vetrov Date: Sun, 4 Aug 2024 00:19:43 +0300 Subject: [PATCH 20/20] Add distinct fragmentation strategies Fix ip4_frag, implement no fragmentation strategy. Related to #24 --- youtubeUnblock.c | 153 +++++++++++++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 58 deletions(-) diff --git a/youtubeUnblock.c b/youtubeUnblock.c index 544f38c..67def6a 100644 --- a/youtubeUnblock.c +++ b/youtubeUnblock.c @@ -27,9 +27,18 @@ #define USE_GSO #endif -#ifndef USE_IP_FRAGMENTATION -#define USE_TCP_SEGMENTATION -#endif +#define FRAG_STRAT_TCP 0 +#define FRAG_STRAT_IP 1 +#define FRAG_STRAT_NONE 2 + +#define FRAGMENTATION_STRATEGY FRAG_STRAT_TCP + +#if FRAGMENTATION_STRATEGY == FRAG_STRAT_TCP + #define USE_TCP_SEGMENTATION +#elif FRAGMENTATION_STRATEGY == FRAG_STRAT_IP + #define USE_IP_FRAGMENTATION +#elif FRAGMENTATION_STRATEGY == FRAG_STRAT_NONE +#endif #define RAWSOCKET_MARK (1 << 15) @@ -173,12 +182,17 @@ static int ipv4_frag(struct pkt_buff *pktb, size_t payload_offset, uint8_t buff2[MNL_SOCKET_BUFFER_SIZE]; struct iphdr *hdr = nfq_ip_get_hdr(pktb); + if (hdr == NULL) { + errno = EINVAL; + return -1; + } + size_t hdr_len = hdr->ihl * 4; uint8_t *payload = pktb_data(pktb) + hdr_len; size_t plen = pktb_len(pktb) - hdr_len; - if (hdr == NULL || payload == NULL || plen <= payload_offset) { + if (payload == NULL || plen <= payload_offset) { errno = EINVAL; return -1; } @@ -255,15 +269,25 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, struct iphdr *hdr = nfq_ip_get_hdr(pktb); size_t hdr_len = hdr->ihl * 4; if (hdr == NULL) { + errno = EINVAL; perror("tcp4_frag: nfq_ip_get_hdr is null\n"); errno = EINVAL; return -1; } - if (hdr->protocol != IPPROTO_TCP || !(ntohs(hdr->frag_off) & IP_DF)) { - perror("tcp4_frag: proto is tcp or request is fragmented"); + if (hdr->protocol != IPPROTO_TCP) { + errno = EINVAL; + perror("tcp4_frag: proto is not tcp"); errno = EINVAL; return -1; } + if (!(ntohs(hdr->frag_off) & IP_DF)) { + errno = EINVAL; + perror("tcp4_frag: request is ip-fragmented"); + fprintf(stderr, "tcp4_frag: frag_off: %04x\n", ntohs(hdr->frag_off)); + errno = EINVAL; + return -1; + } + if (nfq_ip_set_transport_header(pktb, hdr)) { perror("tcp4_frag: ip_set_transport_header"); @@ -274,6 +298,7 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, struct tcphdr *tcph = nfq_tcp_get_hdr(pktb); size_t tcph_len = tcph->doff * 4; if (tcph == NULL) { + errno = EINVAL; perror("tcp4_frag: tcph is NULL"); errno = EINVAL; return -1; @@ -283,6 +308,7 @@ static int tcp4_frag(struct pkt_buff *pktb, size_t payload_offset, size_t plen = nfq_tcp_get_payload_len(tcph, pktb); if (payload == NULL || plen <= payload_offset) { + errno = EINVAL; perror("tcp4_frag: payload is too small or NULL"); errno = EINVAL; return -1; @@ -352,13 +378,18 @@ static int send_raw_socket(struct pkt_buff *pktb) { struct pkt_buff *buff1; struct pkt_buff *buff2; -#ifdef USE_TCP_SEGMENTATION +#if defined(USE_TCP_SEGMENTATION) || defined(RAWSOCK_TCP_FSTRAT) if (tcp4_frag(pktb, AVAILABLE_MTU-128, &buff1, &buff2) < 0) return -1; -#else +#elif defined(USE_IP_FRAGMENTATION) || defined(RAWSOCK_IP_FSTRAT) if (ipv4_frag(pktb, AVAILABLE_MTU-128, &buff1, &buff2) < 0) return -1; - +#else + errno = EINVAL; + printf("send_raw_socket: Packet is too big but fragmentation is disabled! " + "Pass -DRAWSOCK_TCP_FSTRAT or -DRAWSOCK_IP_FSTRAT as CFLAGS " + "To enable it only for raw socket\n"); + return -1; #endif int sent = 0; @@ -452,8 +483,6 @@ static int fallback_accept_packet(uint32_t id) { const char googlevideo_ending[] = "googlevideo.com"; const int googlevideo_len = 15; -#define GOOGLEVIDEO_MARK 0xfc74 - struct verdict { int gvideo_hello; /* google video hello packet */ int sni_offset; /* offset from start of tcp _payload_ */ @@ -718,10 +747,9 @@ static int process_packet(const struct packet_data packet) { struct pkt_buff *frag1; struct pkt_buff *frag2; nfq_nlmsg_verdict_put(verdnlh, packet.id, NF_DROP); - - -#ifdef USE_TCP_SEGMENTATION int ret = 0; + + #ifdef FAKE_SNI struct pkt_buff *fake_sni = gen_fake_sni(ip_header, tcph); if (fake_sni == NULL) { @@ -736,15 +764,13 @@ static int process_packet(const struct packet_data packet) { goto fallback; } #endif - - size_t ipd_offset = vrd.sni_offset; - size_t mid_offset = ipd_offset + vrd.sni_len / 2; - struct pkt_buff *pktb = pktb_alloc( family, packet.payload, packet.payload_len, - 0); + 0); + + if (pktb == NULL) { perror("pktb_alloc of payload"); @@ -753,23 +779,55 @@ static int process_packet(const struct packet_data packet) { #endif goto fallback; } + + + struct iphdr *piph = nfq_ip_get_hdr(pktb); + ret = nfq_ip_set_transport_header(pktb, piph); + struct tcphdr *ptcph = nfq_tcp_get_hdr(pktb); + if (!piph || ret < 0 || !ptcph) { + perror("cannot parse pktb"); + goto gv_free; + } + + nfq_ip_set_checksum(piph); + nfq_tcp_compute_checksum_ipv4(ptcph, piph); + +#if defined(USE_TCP_SEGMENTATION) + size_t ipd_offset = vrd.sni_offset; + size_t mid_offset = ipd_offset + vrd.sni_len / 2; + if (tcp4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { perror("tcp4_frag"); - pktb_free(pktb); -#ifdef FAKE_SNI - pktb_free(fake_sni); -#endif - - goto fallback; + + goto gv_free; } +#elif defined(USE_IP_FRAGMENTATION) + size_t ipd_offset = ((char *)data - (char *)tcph) + vrd.sni_offset; + size_t mid_offset = ipd_offset + vrd.sni_len / 2; + mid_offset += 8 - mid_offset % 8; + + if (ipv4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { + perror("ipv4_frag"); + + goto gv_free; + } + +#else + ret = send_raw_socket(pktb); + if (ret < 0) { + perror("raw pack send"); + } + goto gv_free; +#endif + ret = send_raw_socket(frag2); if (ret < 0) { - errno = ret; perror("raw frags send: frag2"); pktb_free(frag1); - goto err; + + goto free_frags; } #ifdef SEG2_DELAY @@ -782,45 +840,22 @@ static int process_packet(const struct packet_data packet) { #else ret = send_raw_socket(frag1); if (ret < 0) { - errno = ret; perror("raw frags send: frag1"); pktb_free(frag1); - goto err; + + goto free_frags; } + pktb_free(frag1); #endif -err: + +free_frags: pktb_free(frag2); +gv_free: pktb_free(pktb); #ifdef FAKE_SNI pktb_free(fake_sni); #endif - -#else - // TODO: Implement compute of tcp checksum - // GSO may turn kernel to not compute the tcp checksum. - // Also it will never be meaningless to ensure the - // checksum is right. - // nfq_tcp_compute_checksum_ipv4(tcph, ip_header); - - size_t ipd_offset = ((char *)data - (char *)tcph) + vrd.sni_offset; - size_t mid_offset = ipd_offset + vrd.sni_len / 2; - mid_offset += 8 - mid_offset % 8; - - - if (ipv4_frag(pktb, mid_offset, &frag1, &frag2) < 0) { - perror("ipv4_frag"); - goto fallback; - } - - if ((send_raw_socket(frag1) == -1) || (send_raw_socket(frag2) == -1)) { - perror("raw frags send"); - } - - pktb_free(frag1); - pktb_free(frag2); -#endif - } @@ -897,10 +932,12 @@ int main(int argc, const char *argv[]) exit(EXIT_FAILURE); } -#ifdef USE_TCP_SEGMENTATION +#if defined(USE_TCP_SEGMENTATION) printf("Using TCP segmentation\n"); -#else +#elif defined(USE_IP_FRAGMENTATION) printf("Using IP fragmentation\n"); +#else + printf("SNI fragmentation is disabled\n"); #endif #ifdef SEG2_DELAY