Nitter Workaround Issue #983

8 files changed, 218 insertions, 11 deletions

diff --git a/public/posts/ai-snoop-dogg/index.html b/public/posts/ai-snoop-dogg/index.html
index ab3d671..ca58608 100644
--- a/public/posts/ai-snoop-dogg/index.html
+++ b/public/posts/ai-snoop-dogg/index.html
@@ -81,6 +81,8 @@ Our cause so strong, this mission won’t die ! Taking back our rights from
 		<div>
 			<ul>
 				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
 				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
 				
 				<li><a href="/posts/generals/">Generals</a></li>
@@ -89,8 +91,6 @@ Our cause so strong, this mission won&rsquo;t die ! Taking back our rights from
 				
 				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
 				
-				<li><a href="/posts/wolf/">About the Lone Wolf</a></li>
-				
 			</ul>
 		</div>
 	</div>
diff --git a/public/posts/generals/index.html b/public/posts/generals/index.html
index f2bb6b9..0c20500 100644
--- a/public/posts/generals/index.html
+++ b/public/posts/generals/index.html
@@ -595,6 +595,8 @@ Signal, Briar, Element, Session</p>
 		<div>
 			<ul>
 				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
 				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
 				
 				<li><a href="/posts/generals/">Generals</a></li>
@@ -603,8 +605,6 @@ Signal, Briar, Element, Session</p>
 				
 				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
 				
-				<li><a href="/posts/wolf/">About the Lone Wolf</a></li>
-				
 			</ul>
 		</div>
 	</div>
diff --git a/public/posts/index.html b/public/posts/index.html
index f710824..32a79f5 100644
--- a/public/posts/index.html
+++ b/public/posts/index.html
@@ -50,6 +50,14 @@
 		    <li>
 		        <div class="post-title">
 				
+				    2023-10-08 <a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a>
+				
+		        </div>
+		    </li>
+		    
+		    <li>
+		        <div class="post-title">
+				
 				    2023-10-02 <a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a>
 				
 		        </div>
diff --git a/public/posts/index.xml b/public/posts/index.xml
index 7ad5482..08c761f 100644
--- a/public/posts/index.xml
+++ b/public/posts/index.xml
@@ -6,7 +6,18 @@
     <description>Recent content in Posts on blog.tinfoil-hat.net</description>
     <generator>Hugo -- gohugo.io</generator>
     <language>en-us</language>
-    <lastBuildDate>Mon, 02 Oct 2023 04:44:38 +0200</lastBuildDate><atom:link href="https://blog.tinfoil-hat.net/posts/index.xml" rel="self" type="application/rss+xml" />
+    <lastBuildDate>Sun, 08 Oct 2023 00:01:08 +0200</lastBuildDate><atom:link href="https://blog.tinfoil-hat.net/posts/index.xml" rel="self" type="application/rss+xml" />
+    <item>
+      <title>Nitter Ratelimiting &amp; Guest_Accounts branch</title>
+      <link>https://blog.tinfoil-hat.net/posts/nitter-ratelimiting/</link>
+      <pubDate>Sun, 08 Oct 2023 00:01:08 +0200</pubDate>
+      
+      <guid>https://blog.tinfoil-hat.net/posts/nitter-ratelimiting/</guid>
+      <description>Mitigating Unauthorized Web Scraping Bot Traffic The current design of Nitter, along with its methodology for accessing the Twitter service, necessitates heightened vigilance on the part of instance operators to manage unwarranted access by web scraping bots. This guide is intended to provide operators with essential information to effectively mitigate unauthorized web scraping bot traffic.
+Prerequisites Before proceeding with the rate-limiting setup, ensure that you have:
+A functional Nitter installation located at /opt/nitter.</description>
+    </item>
+    
     <item>
       <title>Rammstein Singer Till Lindemann Lizard Confirmed</title>
       <link>https://blog.tinfoil-hat.net/posts/till-lindemann-lizard/</link>
diff --git a/public/posts/nitter-ratelimiting/index.html b/public/posts/nitter-ratelimiting/index.html
new file mode 100644
index 0000000..f52f53f
--- /dev/null
+++ b/public/posts/nitter-ratelimiting/index.html
@@ -0,0 +1,188 @@
+<!DOCTYPE html>
+<html lang="en-us">
+<head>
+	<meta charset="UTF-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1">
+	<meta http-equiv="X-UA-Compatible" content="IE=edge">
+	<style type=text/css>body{font-family:monospace;}</style>
+	<title>Nitter Ratelimiting &amp; Guest_Accounts branch</title>
+	
+	
+	<link rel="stylesheet" href="/css/style.css">
+	
+	
+</head>
+<body>
+	<header>
+	==========================<br>
+	== <a href="https://blog.tinfoil-hat.net">blog.tinfoil-hat.net</a> ==<br>
+	==========================
+	<div style="float: right;">A save home for every tinfoil-hat!</div><br>
+	<p>
+	<nav>
+			<a href="/"><b>Start</b></a>.
+			
+			
+			<a href="/posts/"><b>Posts</b></a>.
+			
+			<a href="/categories/"><b>Categories</b></a>.
+			
+			<a href="/tags/"><b>Tags</b></a>.
+			
+	</nav>
+	</p>
+	<script async defer data-website-id="9b2f4923-27f6-41ac-ba15-43c51d35250e" src="https://stats.tinfoil-hat.net/umami.js"></script>
+	
+</header>
+
+	
+	<main>
+		<article>
+			<h1>Nitter Ratelimiting &amp; Guest_Accounts branch</h1>
+			<b><time>08.10.2023 00:01</time></b>
+		       
+		           <a href="/tags/technology">Technology</a>
+        	       
+		           <a href="/tags/linux">Linux</a>
+        	       
+		           <a href="/tags/server">Server</a>
+        	       
+
+			<div>
+				<h1 id="mitigating-unauthorized-web-scraping-bot-traffic">Mitigating Unauthorized Web Scraping Bot Traffic</h1>
+<p>The current design of Nitter, along with its methodology for accessing the Twitter service, necessitates heightened vigilance on the part of instance operators to manage unwarranted access by web scraping bots. This guide is intended to provide operators with essential information to effectively mitigate unauthorized web scraping bot traffic.</p>
+<h2 id="prerequisites">Prerequisites</h2>
+<p>Before proceeding with the rate-limiting setup, ensure that you have:</p>
+<ol>
+<li>A functional Nitter installation located at <code>/opt/nitter</code>.</li>
+<li>Nginx as your web server, with the server block as outlined in <a href="https://github.com/zedeus/nitter/wiki/Nginx">Nginx Configuration</a>. In this guide, we refer to this server block as &rsquo;nitter.nginx.'</li>
+</ol>
+<h2 id="rate-limiting-configuration">Rate Limiting Configuration</h2>
+<p>Navigate to <code>/etc/nginx</code> and create two necessary files with the following content:</p>
+<p><strong>shared_cache.conf</strong></p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-nginx" data-lang="nginx"><span style="display:flex;"><span><span style="color:#66d9ef">proxy_buffers</span> <span style="color:#ae81ff">64</span> <span style="color:#ae81ff">16k</span>;  
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">proxy_buffer_size</span> <span style="color:#ae81ff">4k</span>;  
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">expires</span> <span style="color:#e6db74">90d</span>;  
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">access_log</span> <span style="color:#66d9ef">off</span>;  
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">proxy_pass</span> <span style="color:#e6db74">http://127.0.0.1:8080</span>;  
+</span></span></code></pre></div><p><strong>shared_static.conf</strong></p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-nginx" data-lang="nginx"><span style="display:flex;"><span><span style="color:#66d9ef">expires</span> <span style="color:#e6db74">90d</span>;
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">access_log</span> <span style="color:#66d9ef">off</span>;
+</span></span><span style="display:flex;"><span><span style="color:#66d9ef">root</span> <span style="color:#e6db74">/opt/nitter/public</span>
+</span></span></code></pre></div><p>These files ensure that normal usage, such as serving images, videos, and site data, does not trigger rate limiting. Logging for these locations is disabled to prevent entries in access or error logs.</p>
+<p>Next, add the rate-limiting rules within the nginx.conf file, located within the http block:</p>
+<p><strong>nginx.conf</strong></p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-nginx" data-lang="nginx"><span style="display:flex;"><span><span style="color:#66d9ef">http</span> {
+</span></span><span style="display:flex;"><span>
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">limit_req_zone</span> $binary_remote_addr <span style="color:#e6db74">zone=nitter.tld_sec:10m</span> <span style="color:#e6db74">rate=1r/s</span>;
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">limit_req_zone</span> $binary_remote_addr <span style="color:#e6db74">zone=nitter.tld_min:10m</span> <span style="color:#e6db74">rate=45r/m</span>;
+</span></span><span style="display:flex;"><span>}
+</span></span></code></pre></div><p>These settings limit users to one request per second and 45 requests per minute, a natural browsing rate for the site.</p>
+<p>Now, in your &rsquo;nitter.nginx&rsquo; server block:</p>
+<p><strong>nitter.nginx</strong></p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-nginx" data-lang="nginx"><span style="display:flex;"><span><span style="color:#66d9ef">server</span> {
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> <span style="color:#e6db74">/pic/</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_cache.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> <span style="color:#e6db74">/video/</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_cache.conf</span>; }
+</span></span><span style="display:flex;"><span>
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> <span style="color:#e6db74">/css/</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> <span style="color:#e6db74">/js/</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> <span style="color:#e6db74">/fonts/</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/apple-touch-icon.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/apple-touch-icon-precomposed.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/android-chrome-192x192.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/favicon-32x32.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/favicon-16x16.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/favicon.ico</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/logo.png</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>        <span style="color:#f92672">location</span> = <span style="color:#e6db74">/site.webmanifest</span> { <span style="color:#f92672">include</span> <span style="color:#e6db74">shared_static.conf</span>; }
+</span></span><span style="display:flex;"><span>}
+</span></span></code></pre></div><p><strong>nitter.nginx</strong></p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-nginx" data-lang="nginx"><span style="display:flex;"><span>        <span style="color:#66d9ef">location</span> <span style="color:#e6db74">/</span> {
+</span></span><span style="display:flex;"><span>                <span style="color:#f92672">proxy_pass</span> <span style="color:#e6db74">http://localhost:8080</span>;
+</span></span><span style="display:flex;"><span>                <span style="color:#f92672">limit_req</span> <span style="color:#e6db74">zone=nitter.tld_sec</span> <span style="color:#e6db74">burst=3</span> <span style="color:#e6db74">nodelay</span>;
+</span></span><span style="display:flex;"><span>                <span style="color:#f92672">limit_req</span> <span style="color:#e6db74">zone=nitter.tld_min</span> <span style="color:#e6db74">burst=4</span>;
+</span></span><span style="display:flex;"><span>        }
+</span></span></code></pre></div><p>The &lsquo;burst&rsquo; parameter allows for temporary bursts of traffic while maintaining the overall rate limit, ensuring a smoother user experience while preventing server overload and misuse.</p>
+<p>Reload Nginx configuration files:</p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>nginx -s reload
+</span></span></code></pre></div><p>Nginx will now rate limit IP for excessive usage.</p>
+<h3 id="fail2ban"><strong>Fail2ban</strong></h3>
+<p>To implement rate limiting and address repeat offenders, a functional install of Fail2ban (<a href="https://github.com/fail2ban/fail2ban">https://github.com/fail2ban/fail2ban</a>) is required. Typically, Fail2ban configuration files are located at /etc/fail2ban. Make a copy of jail.conf named jail.local, as Fail2ban will prioritize jail.local by default when both are present:</p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
+</span></span></code></pre></div><p>Within jail.local, ensure that bantime.increment is uncommented and set to true:</p>
+<pre tabindex="0"><code class="language-fail2ban" data-lang="fail2ban">bantime.increment = true
+</code></pre><p>Additionally, enable the rate-limiting block in jail.local by setting &rsquo;enabled&rsquo; to true:</p>
+<pre tabindex="0"><code class="language-fail2ban" data-lang="fail2ban">[nginx-limit-req]
+enabled = true
+port    = http,https
+logpath = %(nginx_error_log)s
+</code></pre><p>Restart Fail2ban:</p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>systemctl restart fail2ban
+</span></span></code></pre></div><p>To verify that the jail is running:</p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>fail2ban-client status nginx-limit-req
+</span></span></code></pre></div><p>You&rsquo;ll receive a summary of failures triggered by the filter and the number of active actions.</p>
+<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-bash" data-lang="bash"><span style="display:flex;"><span>Status <span style="color:#66d9ef">for</span> the jail: nginx-limit-req
+</span></span><span style="display:flex;"><span>|- Filter
+</span></span><span style="display:flex;"><span>|  |- Currently failed:	<span style="color:#ae81ff">0</span>
+</span></span><span style="display:flex;"><span>|  |- Total failed:	<span style="color:#ae81ff">0</span>
+</span></span><span style="display:flex;"><span>|  <span style="color:#e6db74">`</span>- File list:	%<span style="color:#f92672">(</span>nginx_error_log<span style="color:#f92672">)</span>s
+</span></span><span style="display:flex;"><span><span style="color:#e6db74">`</span>- Actions
+</span></span><span style="display:flex;"><span>   |- Currently banned:	<span style="color:#ae81ff">0</span>
+</span></span><span style="display:flex;"><span>   |- Total banned:	<span style="color:#ae81ff">0</span>
+</span></span><span style="display:flex;"><span>   <span style="color:#e6db74">`</span>- Banned IP list:
+</span></span></code></pre></div><p>Fail2ban will now enforce rate limits, with incremental punishments for repeat infractions when bans expire.</p>
+<hr>
+<hr>
+<h1 id="switch-to-guest_accounts-compile-it-as-normal">Switch to guest_accounts, compile it as normal</h1>
+<pre tabindex="0"><code>git clone https://github.com/zedeus/nitter
+cd nitter
+git branch guest_accounts
+nimble build -d:release
+nimble scss
+nimble md
+</code></pre><pre tabindex="0"><code>touch guest_accounts.json
+vim guest_accounts.json
+</code></pre><p>paste <code>[]</code> inside it</p>
+<h1 id="finally-apply-for-guest-accounts-here">Finally apply for guest accounts here:</h1>
+<p><a href="https://twitterminator.x86-64-unknown-linux-gnu.zip/">https://twitterminator.x86-64-unknown-linux-gnu.zip/</a></p>
+<h2 id="create-cronjob">Create cronjob</h2>
+<pre tabindex="0"><code>0 */4 * * * curl ...guest_accounts.json
+</code></pre>
+			</div>
+		</article>
+	</main>
+<aside>
+	<div>
+		<div>
+			<h3>LATEST POSTS</h3>
+		</div>
+		<div>
+			<ul>
+				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
+				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
+				
+				<li><a href="/posts/generals/">Generals</a></li>
+				
+				<li><a href="/posts/prep/">Prepping</a></li>
+				
+				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
+				
+			</ul>
+		</div>
+	</div>
+</aside>
+
+
+	<footer>
+	<p>&copy; 2023 <a href="https://blog.tinfoil-hat.net"><b>blog.tinfoil-hat.net</b></a>.
+	<a href="https://tinfoil-hat.net"><b>Landing Page</b></a>.
+	<a href="http://keys.gnupg.net/pks/lookup?op=vindex&amp;fingerprint=on&amp;search=0xC0090D40CFE34966"><b>GPG-Fingerprint</b></a>.
+	<a href="mailto:mal@tinfoil-hat.net"><b>E-Mail</b></a>.
+	<a href="https://git.tinfoil-hat.net"><b>Gitweb</b></a>.
+	</p>
+</footer>
+
+</body>
+</html>
diff --git a/public/posts/prep/index.html b/public/posts/prep/index.html
index 9c7313a..e1f2b3d 100644
--- a/public/posts/prep/index.html
+++ b/public/posts/prep/index.html
@@ -128,6 +128,8 @@
 		<div>
 			<ul>
 				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
 				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
 				
 				<li><a href="/posts/generals/">Generals</a></li>
@@ -136,8 +138,6 @@
 				
 				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
 				
-				<li><a href="/posts/wolf/">About the Lone Wolf</a></li>
-				
 			</ul>
 		</div>
 	</div>
diff --git a/public/posts/till-lindemann-lizard/index.html b/public/posts/till-lindemann-lizard/index.html
index 1c9f21e..d5975e3 100644
--- a/public/posts/till-lindemann-lizard/index.html
+++ b/public/posts/till-lindemann-lizard/index.html
@@ -72,6 +72,8 @@
 		<div>
 			<ul>
 				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
 				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
 				
 				<li><a href="/posts/generals/">Generals</a></li>
@@ -80,8 +82,6 @@
 				
 				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
 				
-				<li><a href="/posts/wolf/">About the Lone Wolf</a></li>
-				
 			</ul>
 		</div>
 	</div>
diff --git a/public/posts/wolf/index.html b/public/posts/wolf/index.html
index d30592c..dd7b941 100644
--- a/public/posts/wolf/index.html
+++ b/public/posts/wolf/index.html
@@ -339,6 +339,8 @@ up the pitchers we drink!&rdquo;</p>
 		<div>
 			<ul>
 				
+				<li><a href="/posts/nitter-ratelimiting/">Nitter Ratelimiting &amp; Guest_Accounts branch</a></li>
+				
 				<li><a href="/posts/till-lindemann-lizard/">Rammstein Singer Till Lindemann Lizard Confirmed</a></li>
 				
 				<li><a href="/posts/generals/">Generals</a></li>
@@ -347,8 +349,6 @@ up the pitchers we drink!&rdquo;</p>
 				
 				<li><a href="/posts/ai-snoop-dogg/">Ai Snoop Dogg</a></li>
 				
-				<li><a href="/posts/wolf/">About the Lone Wolf</a></li>
-				
 			</ul>
 		</div>
 	</div>