Merge remote-tracking branch 'origin/vnext' into vnext

# Conflicts: # DragonFruit.OnionFruit.Web.Worker/Generators/OnionDbGenerator.cs
dragonfruitnetwork · Oct 14, 2023 · 930fc52 · 930fc52
2 parents 2888a7f + e0ebef0
commit 930fc52
Show file tree

Hide file tree

Showing 42 changed files with 627 additions and 173 deletions.
diff --git a/DragonFruit.OnionFruit.Web.Worker/Database/OnionFruitNodeInfo.cs b/DragonFruit.OnionFruit.Web.Worker/Database/OnionFruitNodeInfo.cs
@@ -1,9 +1,12 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using DragonFruit.OnionFruit.Web.Worker.Sources.Onionoo.Enums;
 using Redis.OM.Modeling;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Database;
 
-[Document(Prefixes = new[] { "yuna:onionfruit-nodes" }, IndexName = "yuna:onionfruit-nodes-idx", StorageType = StorageType.Json)]
+[Document(Prefixes = new[] {"yuna:onionfruit-nodes"}, IndexName = "yuna:onionfruit-nodes-idx", StorageType = StorageType.Json)]
 public class OnionFruitNodeInfo
 {
     [Indexed]

diff --git a/DragonFruit.OnionFruit.Web.Worker/DragonFruit.OnionFruit.Web.Worker.csproj b/DragonFruit.OnionFruit.Web.Worker/DragonFruit.OnionFruit.Web.Worker.csproj
@@ -7,26 +7,26 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="DnsClient" Version="1.7.0" />
-        <PackageReference Include="DragonFruit.Data.Serializers.SystemJson" Version="2023.727.0" />
-        <PackageReference Include="DragonFruit.OnionFruit.Web.Worker.Native" Version="0.2.0" />
-        <PackageReference Include="Google.Protobuf" Version="3.18.0" />
-        <PackageReference Include="Grpc.Tools" Version="2.40.0">
+        <PackageReference Include="DnsClient" Version="1.7.0"/>
+        <PackageReference Include="DragonFruit.Data.Serializers.SystemJson" Version="2023.727.0"/>
+        <PackageReference Include="DragonFruit.OnionFruit.Web.Worker.Native" Version="0.2.0"/>
+        <PackageReference Include="Google.Protobuf" Version="3.24.4" />
+        <PackageReference Include="Grpc.Tools" Version="2.58.0">
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
             <PrivateAssets>all</PrivateAssets>
         </PackageReference>
-        <PackageReference Include="IPAddressRange" Version="6.0.0" />
-        <PackageReference Include="JetBrains.Annotations" Version="2023.2.0" />
-        <PackageReference Include="libloc.Access" Version="2023.1006.0" />
+        <PackageReference Include="IPAddressRange" Version="6.0.0"/>
+        <PackageReference Include="JetBrains.Annotations" Version="2023.2.0"/>
+        <PackageReference Include="libloc.Access" Version="2023.1013.0" />
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1"/>
-        <PackageReference Include="Redis.OM" Version="0.5.3" />
+        <PackageReference Include="Redis.OM" Version="0.5.3"/>
     </ItemGroup>
-    
+
     <ItemGroup>
-        <Protobuf Include="..\oniondb.proto" GrpcServices="Server" />
+        <Protobuf Include="..\oniondb.proto" GrpcServices="Server"/>
         <Content Include="..\.dockerignore">
-        <Link>.dockerignore</Link>
-    </Content>
+            <Link>.dockerignore</Link>
+        </Content>
     </ItemGroup>
-    
+
 </Project>
diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/ClientCountriesDatabaseGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/ClientCountriesDatabaseGenerator.cs
@@ -1,5 +1,5 @@
-// OnionFruit Web Copyright DragonFruit Network
-// Licensed under the MIT License. Please refer to the LICENSE file at the root of this project for details
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
 
 using System.Collections.Generic;
 using System.Linq;
@@ -8,7 +8,7 @@
 using DragonFruit.OnionFruit.Web.Worker.Sources;
 using DragonFruit.OnionFruit.Web.Worker.Sources.Onionoo;
 using DragonFruit.OnionFruit.Web.Worker.Sources.Onionoo.Enums;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
+using DragonFruit.OnionFruit.Web.Worker.Storage.Abstractions;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Generators;
 
@@ -29,12 +29,12 @@ public async Task GenerateDatabase(IFileSink fileSink)
             ["in"] = CountriesWithFlag(_torInfo.Countries, TorNodeFlags.Guard),
             ["out"] = CountriesWithFlag(_torInfo.Countries, TorNodeFlags.Exit)
         };
-        
+
         await fileSink.CreateFile("legacy/countries.json").WriteAsync(JsonSerializer.SerializeToUtf8Bytes(clientData)).ConfigureAwait(false);
     }
 
     private static IEnumerable<string> CountriesWithFlag(IEnumerable<IGrouping<string, TorRelayDetails>> info, TorNodeFlags flag)
     {
         return info.Where(x => x.Any(y => y.Flags.HasFlag(flag))).Select(x => x.Key);
     }
-}
+}
diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/ClientGeoIpFileGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/ClientGeoIpFileGenerator.cs
@@ -1,10 +1,13 @@
-using System.IO;
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
+using System.IO;
 using System.Linq;
 using System.Net;
 using System.Text;
 using System.Threading.Tasks;
 using DragonFruit.OnionFruit.Web.Worker.Sources;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
+using DragonFruit.OnionFruit.Web.Worker.Storage.Abstractions;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Generators;
 

diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/IDatabaseGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/IDatabaseGenerator.cs
@@ -1,5 +1,8 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System.Threading.Tasks;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
+using DragonFruit.OnionFruit.Web.Worker.Storage.Abstractions;
 using JetBrains.Annotations;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Generators;

diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/OnionDbGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/OnionDbGenerator.cs
@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using System.Threading.Tasks;
 using DragonFruit.OnionFruit.Web.Worker.Sources;
 using DragonFruit.OnionFruit.Web.Worker.Sources.Onionoo.Enums;
@@ -13,29 +12,29 @@ namespace DragonFruit.OnionFruit.Web.Worker.Generators;
 
 public class OnionDbGenerator : IDatabaseGenerator
 {
-    protected readonly OnionooDataSource Onionoo;
-    protected readonly LocationDbSource LocationDb;
+    private readonly OnionooDataSource _onionoo;
+    private readonly LocationDbSource _locationDb;
 
     public OnionDbGenerator(OnionooDataSource onionoo, LocationDbSource locationDb)
     {
-        Onionoo = onionoo;
-        LocationDb = locationDb;
+        _onionoo = onionoo;
+        _locationDb = locationDb;
     }
-    
+
     public Task GenerateDatabase(IFileSink fileSink)
     {
         var database = CreateBaseDb();
-        
+
         // iterate through each country and write info
-        foreach (var country in LocationDb.Database.Countries)
+        foreach (var country in _locationDb.Database.Countries)
         {
-            var nodes = Onionoo.Countries.SingleOrDefault(x => x.Key.Equals(country.Code, StringComparison.OrdinalIgnoreCase));
+            var nodes = _onionoo.Countries.SingleOrDefault(x => x.Key.Equals(country.Code, StringComparison.OrdinalIgnoreCase));
 
             if (nodes == null)
             {
                 continue;
             }
-            
+
             var countryData = new OnionDbCountry
             {
                 CountryCode = country.Code,
@@ -54,11 +53,11 @@ public Task GenerateDatabase(IFileSink fileSink)
                     case nameof(TorNodeFlags.Guard):
                         countryData.EntryNodeCount++;
                         break;
-                    
+
                     case nameof(TorNodeFlags.Fast):
                         countryData.FastNodeCount++;
                         break;
-                    
+
                     case nameof(TorNodeFlags.Running):
                         countryData.OnlineNodeCount++;
                         break;
@@ -67,22 +66,26 @@ public Task GenerateDatabase(IFileSink fileSink)
                 countryData.TotalNodeCount++;
             }
 
-            WriteCountryAddressRanges(countryData, LocationDb.IPv4CountryAddressRanges[country.Code], LocationDb.IPv6CountryAddressRanges[country.Code]);
+            WriteCountryAddressRanges(countryData, _locationDb.IPv4CountryAddressRanges[country.Code], _locationDb.IPv6CountryAddressRanges[country.Code]);
             database.Countries.Add(countryData);
         }
 
         OnDatabaseGenerated(fileSink, database);
         return Task.CompletedTask;
     }
-    
+
     protected virtual OnionDb CreateBaseDb() => new()
     {
-        DbFormatVersion = 1,
-        MagicBytes = ByteString.CopyFrom(0xDB),
-        GeoLicense = LocationDb.Database.License,
+        MagicBytes = ByteString.CopyFrom(0xDB, 0x01),
+
+        // tor license can be found at the footer of https://metrics.torproject.org/onionoo.html
+        TorLicense = "CC0",
+        GeoLicense = _locationDb.Database.License,
+
+        // versions derived from datasets/current epoch
         DbVersion = DateTimeOffset.UtcNow.ToUnixTimeSeconds(),
-        TorDirVersion = Onionoo.DataLastModified.ToUnixTimeSeconds(),
-        GeoDirVersion = new DateTimeOffset(LocationDb.Database.CreatedAt).ToUnixTimeSeconds()
+        TorDirVersion = _onionoo.DataLastModified.ToUnixTimeSeconds(),
+        GeoDirVersion = _locationDb.Database.CreatedAt.ToUnixTimeSeconds()
     };
 
     protected virtual void WriteCountryAddressRanges(OnionDbCountry country, IEnumerable<IPAddressRange> v4Ranges, IEnumerable<IPAddressRange> v6Ranges)

diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/OnionLiteDbGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/OnionLiteDbGenerator.cs
@@ -1,6 +1,9 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System.Collections.Generic;
 using DragonFruit.OnionFruit.Web.Worker.Sources;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
+using DragonFruit.OnionFruit.Web.Worker.Storage.Abstractions;
 using Google.Protobuf;
 using NetTools;
 
@@ -15,11 +18,11 @@ public OnionLiteDbGenerator(OnionooDataSource onionoo, LocationDbSource location
         : base(onionoo, locationDb)
     {
     }
-    
+
     protected override OnionDb CreateBaseDb()
     {
         var baseDb = base.CreateBaseDb();
-        
+
         baseDb.ClearGeoLicense();
         baseDb.ClearGeoDirVersion();
 

diff --git a/DragonFruit.OnionFruit.Web.Worker/Generators/RedisDatabaseGenerator.cs b/DragonFruit.OnionFruit.Web.Worker/Generators/RedisDatabaseGenerator.cs
@@ -1,9 +1,12 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System;
 using System.Linq;
 using System.Threading.Tasks;
 using DragonFruit.OnionFruit.Web.Worker.Database;
 using DragonFruit.OnionFruit.Web.Worker.Sources;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
+using DragonFruit.OnionFruit.Web.Worker.Storage.Abstractions;
 using Redis.OM.Contracts;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Generators;
@@ -23,7 +26,7 @@ public async Task GenerateDatabase(IFileSink fileSink)
     {
         var table = _redis.RedisCollection<OnionFruitNodeInfo>();
         var dbVersion = (long)_torDirectory.DataLastModified.Subtract(DateTime.UnixEpoch).TotalSeconds;
-        
+
         foreach (var relay in _torDirectory.Relays)
         {
             // drop the port off the addresses

diff --git a/DragonFruit.OnionFruit.Web.Worker/Native/NativeMethods.cs b/DragonFruit.OnionFruit.Web.Worker/Native/NativeMethods.cs
@@ -1,4 +1,7 @@
-using System.Runtime.InteropServices;
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
+using System.Runtime.InteropServices;
 
 namespace DragonFruit.OnionFruit.Web.Worker.Native;
 

diff --git a/DragonFruit.OnionFruit.Web.Worker/Native/NativeStructs.cs b/DragonFruit.OnionFruit.Web.Worker/Native/NativeStructs.cs
@@ -1,6 +1,10 @@
-using System;
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
+using System;
 using System.Runtime.InteropServices;
 
+// ReSharper disable InconsistentNaming
 namespace DragonFruit.OnionFruit.Web.Worker.Native;
 
 [StructLayout(LayoutKind.Sequential, Pack = 1)]
@@ -26,8 +30,9 @@ public struct NetworkEntry
 {
     [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)]
     public byte[] network;
+
     public byte cidr;
-        
+
     [MarshalAs(UnmanagedType.ByValArray, SizeConst = 2)]
     public byte[] country_code;
 }

diff --git a/DragonFruit.OnionFruit.Web.Worker/Program.cs b/DragonFruit.OnionFruit.Web.Worker/Program.cs
@@ -1,11 +1,13 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System;
 using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
 using DnsClient;
 using DragonFruit.Data;
-using DragonFruit.OnionFruit.Web.Worker.Storage;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -67,7 +69,7 @@ private static void ConfigureServices(HostBuilderContext context, IServiceCollec
         // redis + redis.om
         services.AddSingleton(RedisClientConfigurator.CreateConnectionMultiplexer(context.Configuration, true));
         services.AddSingleton<IRedisConnectionProvider>(s => new RedisConnectionProvider(s.GetRequiredService<IConnectionMultiplexer>()));
-            
+
         // api
         services.AddSingleton<ILookupClient, LookupClient>();
         services.AddSingleton<ApiClient, WorkerApiClient>();

diff --git a/...Worker/Storage/RedisClientConfigurator.cs → ...uit.Web.Worker/RedisClientConfigurator.cs b/...Worker/Storage/RedisClientConfigurator.cs → ...uit.Web.Worker/RedisClientConfigurator.cs
@@ -1,8 +1,11 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System;
 using Microsoft.Extensions.Configuration;
 using StackExchange.Redis;
 
-namespace DragonFruit.OnionFruit.Web.Worker.Storage;
+namespace DragonFruit.OnionFruit.Web.Worker;
 
 public static class RedisClientConfigurator
 {

diff --git a/DragonFruit.OnionFruit.Web.Worker/Sources/IDataSource.cs b/DragonFruit.OnionFruit.Web.Worker/Sources/IDataSource.cs
@@ -1,3 +1,6 @@
+// OnionFruit™ Web Copyright DragonFruit Network <[email protected]>
+// Licensed under Apache-2. Refer to the LICENSE file for more info
+
 using System;
 using System.Threading.Tasks;
 using JetBrains.Annotations;