Cleanup VirusTotal scanner

VoidCat/Controllers/StatsController.cs

@@ -29,7 +29,8 @@ namespace VoidCat.Controllers
                 bytes += vf.Metadata?.Size ?? 0;
                 count++;
             }
-            return new(bw, bytes, count);
+
+            return new(bw, bytes, count, BuildInfo.GetBuildInfo());
         }
 
         [HttpGet]
@@ -41,6 +42,7 @@ namespace VoidCat.Controllers
         }
     }
 
-    public sealed record GlobalStats(Bandwidth Bandwidth, ulong TotalBytes, int Count);
+    public sealed record GlobalStats(Bandwidth Bandwidth, ulong TotalBytes, int Count, BuildInfo BuildInfo);
+
     public sealed record FileStats(Bandwidth Bandwidth);
-}
+}

VoidCat/Model/BuildInfo.cs

@@ -0,0 +1,31 @@
+using System.Reflection;
+
+namespace VoidCat.Model;
+
+public class BuildInfo
+{
+    public string? Version { get; init; }
+    public string? GitHash { get; init; }
+    public DateTime BuildTime { get; init; }
+
+    public static BuildInfo GetBuildInfo()
+    {
+        var asm = Assembly.GetEntryAssembly();
+        var version = asm.GetName().Version;
+
+        var gitHash = asm
+            .GetCustomAttributes<AssemblyMetadataAttribute>()
+            .FirstOrDefault(attr => attr.Key == "GitHash")?.Value;
+
+        var buildTime = asm
+            .GetCustomAttributes<AssemblyMetadataAttribute>()
+            .FirstOrDefault(attr => attr.Key == "BuildTime");
+
+        return new()
+        {
+            Version = $"{version.Major}.{version.Minor}.{version.Build}",
+            GitHash = gitHash,
+            BuildTime = DateTime.FromBinary(long.Parse(buildTime?.Value ?? "0"))
+        };
+    }
+}

VoidCat/Services/Background/VirusScannerService.cs

@@ -1,4 +1,5 @@
 using VoidCat.Services.Abstractions;
+using VoidCat.Services.VirusScanner.Exceptions;
 
 namespace VoidCat.Services.Background;
 
@@ -42,6 +43,12 @@ public class VirusScannerService : BackgroundService
                             var result = await _scanner.ScanFile(file.Id, stoppingToken);
                             await _scanStore.Set(file.Id, result);
                         }
+                        catch (RateLimitedException rx)
+                        {
+                            var sleep = rx.RetryAfter ?? DateTimeOffset.UtcNow.AddMinutes(10);
+                            _logger.LogWarning("VirusScanner was rate limited, sleeping until {Time}", sleep);
+                            await Task.Delay(sleep - DateTimeOffset.UtcNow, stoppingToken);
+                        }
                         catch (Exception ex)
                         {
                             _logger.LogError(ex, "Failed to scan file {Id} error={Message}", file.Id, ex.Message);

VoidCat/Services/VirusScanner/Exceptions/RateLimitedException.cs

@@ -0,0 +1,6 @@
+namespace VoidCat.Services.VirusScanner.Exceptions;
+
+public class RateLimitedException : Exception
+{
+    public DateTimeOffset? RetryAfter { get; init; }
+}

VoidCat/Services/VirusScanner/VirusTotal/VirusTotalClasses.cs

@@ -1,285 +1,133 @@
-using System.Runtime.Serialization;
-using Newtonsoft.Json;
+using Newtonsoft.Json;
+// ReSharper disable InconsistentNaming
+#pragma warning disable CS8618
 
 namespace VoidCat.Services.VirusScanner.VirusTotal;
 
-// Root myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
-public class AlertContext
-{
-    [JsonProperty("proto")] public string Proto { get; set; }
-
-    [JsonProperty("src_ip")] public string SrcIp { get; set; }
-
-    [JsonProperty("src_port")] public int SrcPort { get; set; }
-}
-
-public class CrowdsourcedIdsResult
-{
-    [JsonProperty("alert_context")] public List<AlertContext> AlertContext { get; set; }
-
-    [JsonProperty("alert_severity")] public string AlertSeverity { get; set; }
-
-    [JsonProperty("rule_category")] public string RuleCategory { get; set; }
-
-    [JsonProperty("rule_id")] public string RuleId { get; set; }
-
-    [JsonProperty("rule_msg")] public string RuleMsg { get; set; }
-
-    [JsonProperty("rule_source")] public string RuleSource { get; set; }
-}
-
-public class CrowdsourcedIdsStats
-{
-    [JsonProperty("high")] public int High { get; set; }
-
-    [JsonProperty("info")] public int Info { get; set; }
-
-    [JsonProperty("low")] public int Low { get; set; }
-
-    [JsonProperty("medium")] public int Medium { get; set; }
-}
-
-public class CrowdsourcedYaraResult
-{
-    [JsonProperty("description")] public string Description { get; set; }
-
-    [JsonProperty("match_in_subfile")] public bool MatchInSubfile { get; set; }
-
-    [JsonProperty("rule_name")] public string RuleName { get; set; }
-
-    [JsonProperty("ruleset_id")] public string RulesetId { get; set; }
-
-    [JsonProperty("ruleset_name")] public string RulesetName { get; set; }
-
-    [JsonProperty("source")] public string Source { get; set; }
-}
-
-public class ALYac
-{
-    [JsonProperty("category")] public string Category { get; set; }
-
-    [JsonProperty("engine_name")] public string EngineName { get; set; }
-
-    [JsonProperty("engine_update")] public string EngineUpdate { get; set; }
-
-    [JsonProperty("engine_version")] public string EngineVersion { get; set; }
-
-    [JsonProperty("method")] public string Method { get; set; }
-
-    [JsonProperty("result")] public string Result { get; set; }
-}
-
-public class APEX
-{
-    [JsonProperty("category")] public string Category { get; set; }
-
-    [JsonProperty("engine_name")] public string EngineName { get; set; }
-
-    [JsonProperty("engine_update")] public string EngineUpdate { get; set; }
-
-    [JsonProperty("engine_version")] public string EngineVersion { get; set; }
-
-    [JsonProperty("method")] public string Method { get; set; }
-
-    [JsonProperty("result")] public string Result { get; set; }
-}
-
-public class AVG
-{
-    [JsonProperty("category")] public string Category { get; set; }
-
-    [JsonProperty("engine_name")] public string EngineName { get; set; }
-
-    [JsonProperty("engine_update")] public string EngineUpdate { get; set; }
-
-    [JsonProperty("engine_version")] public string EngineVersion { get; set; }
-
-    [JsonProperty("method")] public string Method { get; set; }
-
-    [JsonProperty("result")] public string Result { get; set; }
-}
-
-public class Acronis
-{
-    [JsonProperty("category")] public string Category { get; set; }
-
-    [JsonProperty("engine_name")] public string EngineName { get; set; }
-
-    [JsonProperty("engine_update")] public string EngineUpdate { get; set; }
-
-    [JsonProperty("engine_version")] public string EngineVersion { get; set; }
-
-    [JsonProperty("method")] public string Method { get; set; }
-
-    [JsonProperty("result")] public object Result { get; set; }
-}
-
-public class LastAnalysisResults
-{
-    [JsonProperty("ALYac")] public ALYac ALYac { get; set; }
-
-    [JsonProperty("APEX")] public APEX APEX { get; set; }
-
-    [JsonProperty("AVG")] public AVG AVG { get; set; }
-
-    [JsonProperty("Acronis")] public Acronis Acronis { get; set; }
-}
-
 public class LastAnalysisStats
 {
-    [JsonProperty("confirmed-timeout")] public int ConfirmedTimeout { get; set; }
+    [JsonProperty("confirmed-timeout")] 
+    public int ConfirmedTimeout { get; set; }
 
-    [JsonProperty("failure")] public int Failure { get; set; }
+    [JsonProperty("failure")] 
+    public int Failure { get; set; }
 
-    [JsonProperty("harmless")] public int Harmless { get; set; }
+    [JsonProperty("harmless")] 
+    public int Harmless { get; set; }
 
-    [JsonProperty("malicious")] public int Malicious { get; set; }
+    [JsonProperty("malicious")] 
+    public int Malicious { get; set; }
 
-    [JsonProperty("suspicious")] public int Suspicious { get; set; }
+    [JsonProperty("suspicious")] 
+    public int Suspicious { get; set; }
 
-    [JsonProperty("timeout")] public int Timeout { get; set; }
+    [JsonProperty("timeout")] 
+    public int Timeout { get; set; }
 
-    [JsonProperty("type-unsupported")] public int TypeUnsupported { get; set; }
+    [JsonProperty("type-unsupported")] 
+    public int TypeUnsupported { get; set; }
 
-    [JsonProperty("undetected")] public int Undetected { get; set; }
-}
-
-public class VirusTotalJujubox
-{
-    [JsonProperty("category")] public string Category { get; set; }
-
-    [JsonProperty("confidence")] public int Confidence { get; set; }
-
-    [JsonProperty("malware_classification")]
-    public List<string> MalwareClassification { get; set; }
-
-    [JsonProperty("malware_names")] public List<string> MalwareNames { get; set; }
-
-    [JsonProperty("sandbox_name")] public string SandboxName { get; set; }
-}
-
-public class SandboxVerdicts
-{
-    [JsonProperty("VirusTotal Jujubox")] public VirusTotalJujubox VirusTotalJujubox { get; set; }
-}
-
-public class SigmaAnalysisStats
-{
-    [JsonProperty("critical")] public int Critical { get; set; }
-
-    [JsonProperty("high")] public int High { get; set; }
-
-    [JsonProperty("low")] public int Low { get; set; }
-
-    [JsonProperty("medium")] public int Medium { get; set; }
-}
-
-public class SigmaIntegratedRuleSetGitHub
-{
-    [JsonProperty("critical")] public int Critical { get; set; }
-
-    [JsonProperty("high")] public int High { get; set; }
-
-    [JsonProperty("low")] public int Low { get; set; }
-
-    [JsonProperty("medium")] public int Medium { get; set; }
-}
-
-public class SigmaAnalysisSummary
-{
-    [JsonProperty("Sigma Integrated Rule Set (GitHub)")]
-    public SigmaIntegratedRuleSetGitHub SigmaIntegratedRuleSetGitHub { get; set; }
+    [JsonProperty("undetected")] 
+    public int Undetected { get; set; }
 }
 
 public class TotalVotes
 {
-    [JsonProperty("harmless")] public int Harmless { get; set; }
+    [JsonProperty("harmless")] 
+    public int Harmless { get; set; }
 
-    [JsonProperty("malicious")] public int Malicious { get; set; }
+    [JsonProperty("malicious")] 
+    public int Malicious { get; set; }
 }
 
 public class Attributes
 {
-    [JsonProperty("capabilities_tags")] public List<string> CapabilitiesTags { get; set; }
+    [JsonProperty("capabilities_tags")] 
+    public List<string> CapabilitiesTags { get; set; }
 
-    [JsonProperty("creation_date")] public int CreationDate { get; set; }
+    [JsonProperty("creation_date")] 
+    public int CreationDate { get; set; }
 
-    [JsonProperty("crowdsourced_ids_results")]
-    public List<CrowdsourcedIdsResult> CrowdsourcedIdsResults { get; set; }
-
-    [JsonProperty("crowdsourced_ids_stats")]
-    public CrowdsourcedIdsStats CrowdsourcedIdsStats { get; set; }
-
-    [JsonProperty("crowdsourced_yara_results")]
-    public List<CrowdsourcedYaraResult> CrowdsourcedYaraResults { get; set; }
-
-    [JsonProperty("downloadable")] public bool Downloadable { get; set; }
+    [JsonProperty("downloadable")] 
+    public bool Downloadable { get; set; }
 
     [JsonProperty("first_submission_date")]
     public int FirstSubmissionDate { get; set; }
 
-    [JsonProperty("last_analysis_date")] public int LastAnalysisDate { get; set; }
+    [JsonProperty("last_analysis_date")] 
+    public int LastAnalysisDate { get; set; }
 
-    [JsonProperty("last_analysis_results")]
-    public LastAnalysisResults LastAnalysisResults { get; set; }
-
-    [JsonProperty("last_analysis_stats")] public LastAnalysisStats LastAnalysisStats { get; set; }
+    [JsonProperty("last_analysis_stats")] 
+    public LastAnalysisStats LastAnalysisStats { get; set; }
 
     [JsonProperty("last_modification_date")]
     public int LastModificationDate { get; set; }
 
-    [JsonProperty("last_submission_date")] public int LastSubmissionDate { get; set; }
+    [JsonProperty("last_submission_date")] 
+    public int LastSubmissionDate { get; set; }
 
-    [JsonProperty("md5")] public string Md5 { get; set; }
+    [JsonProperty("md5")] 
+    public string Md5 { get; set; }
 
-    [JsonProperty("meaningful_name")] public string MeaningfulName { get; set; }
+    [JsonProperty("meaningful_name")] 
+    public string MeaningfulName { get; set; }
 
-    [JsonProperty("names")] public List<string> Names { get; set; }
+    [JsonProperty("names")] 
+    public List<string> Names { get; set; }
 
-    [JsonProperty("reputation")] public int Reputation { get; set; }
+    [JsonProperty("reputation")] 
+    public int Reputation { get; set; }
 
-    [JsonProperty("sandbox_verdicts")] public SandboxVerdicts SandboxVerdicts { get; set; }
+    [JsonProperty("sha1")] 
+    public string Sha1 { get; set; }
 
-    [JsonProperty("sha1")] public string Sha1 { get; set; }
+    [JsonProperty("sha256")] 
+    public string Sha256 { get; set; }
 
-    [JsonProperty("sha256")] public string Sha256 { get; set; }
+    [JsonProperty("size")] 
+    public int Size { get; set; }
 
-    [JsonProperty("sigma_analysis_stats")] public SigmaAnalysisStats SigmaAnalysisStats { get; set; }
+    [JsonProperty("tags")] 
+    public List<string> Tags { get; set; }
 
-    [JsonProperty("sigma_analysis_summary")]
-    public SigmaAnalysisSummary SigmaAnalysisSummary { get; set; }
+    [JsonProperty("times_submitted")] 
+    public int TimesSubmitted { get; set; }
 
-    [JsonProperty("size")] public int Size { get; set; }
+    [JsonProperty("total_votes")] 
+    public TotalVotes TotalVotes { get; set; }
 
-    [JsonProperty("tags")] public List<string> Tags { get; set; }
+    [JsonProperty("type_description")] 
+    public string TypeDescription { get; set; }
 
-    [JsonProperty("times_submitted")] public int TimesSubmitted { get; set; }
+    [JsonProperty("type_tag")] 
+    public string TypeTag { get; set; }
 
-    [JsonProperty("total_votes")] public TotalVotes TotalVotes { get; set; }
+    [JsonProperty("unique_sources")] 
+    public int UniqueSources { get; set; }
 
-    [JsonProperty("type_description")] public string TypeDescription { get; set; }
-
-    [JsonProperty("type_tag")] public string TypeTag { get; set; }
-
-    [JsonProperty("unique_sources")] public int UniqueSources { get; set; }
-
-    [JsonProperty("vhash")] public string Vhash { get; set; }
+    [JsonProperty("vhash")] 
+    public string Vhash { get; set; }
 }
 
 public class Links
 {
-    [JsonProperty("self")] public string Self { get; set; }
+    [JsonProperty("self")] 
+    public string Self { get; set; }
 }
 
 public class File
 {
-    [JsonProperty("attributes")] public Attributes Attributes { get; set; }
+    [JsonProperty("attributes")] 
+    public Attributes Attributes { get; set; }
 
-    [JsonProperty("id")] public string Id { get; set; }
+    [JsonProperty("id")] 
+    public string Id { get; set; }
 
-    [JsonProperty("links")] public Links Links { get; set; }
+    [JsonProperty("links")] 
+    public Links Links { get; set; }
 
-    [JsonProperty("type")] public string Type { get; set; }
+    [JsonProperty("type")] 
+    public string Type { get; set; }
 }
 
 public class Error
@@ -305,23 +153,32 @@ public class VTException : Exception
 {
     public VTException(Error error)
     {
-        Error = error;
+        ErrorCode = Enum.TryParse<VTErrorCodes>(error.Code, out var c) ? c : VTErrorCodes.UnknownError;
+        Message = error.Message;
     }
 
-    protected VTException(SerializationInfo info, StreamingContext context, Error error) : base(info, context)
-    {
-        Error = error;
-    }
+    public VTErrorCodes ErrorCode { get; }
+    
+    public string Message { get; }
+}
 
-    public VTException(string? message, Error error) : base(message)
-    {
-        Error = error;
-    }
-
-    public VTException(string? message, Exception? innerException, Error error) : base(message, innerException)
-    {
-        Error = error;
-    }
-
-    public Error Error { get; }
+public enum VTErrorCodes
+{
+    UnknownError,
+    BadRequestError,
+    InvalidArgumentError,
+    NotAvailableYet,
+    UnselectiveContentQueryError,
+    UnsupportedContentQueryError,
+    AuthenticationRequiredError,
+    UserNotActiveError,
+    WrongCredentialsError,
+    ForbiddenError,
+    NotFoundError,
+    AlreadyExistsError,
+    FailedDependencyError,
+    QuotaExceededError,
+    TooManyRequestsError,
+    TransientError,
+    DeadlineExceededError
 }

VoidCat/Services/VirusScanner/VirusTotal/VirusTotalScanner.cs

@@ -1,6 +1,7 @@
 using System.Security.Cryptography;
 using VoidCat.Model;
 using VoidCat.Services.Abstractions;
+using VoidCat.Services.VirusScanner.Exceptions;
 
 namespace VoidCat.Services.VirusScanner.VirusTotal;
 
@@ -24,13 +25,30 @@ public class VirusTotalScanner : IVirusScanner
         // hash file and check on VT
         var hash = await SHA256.Create().ComputeHashAsync(fs, cts);
 
-        var report = await _client.GetReport(hash.ToHex());
-        if (report != default)
+        try
         {
-            return new()
+            var report = await _client.GetReport(hash.ToHex());
+            if (report != default)
             {
-                IsVirus = report.Attributes.Reputation == 0
-            };
+                return new()
+                {
+                    IsVirus = report.Attributes.Reputation == 0
+                };
+            }
+        }
+        catch (VTException vx)
+        {
+            if (vx.ErrorCode == VTErrorCodes.QuotaExceededError)
+            {
+                throw new RateLimitedException()
+                {
+                    // retry tomorrow :( 
+                    // this makes it pretty much unusable unless you have a paid subscription
+                    RetryAfter = DateTimeOffset.Now.Date.AddDays(1)
+                };
+            }
+
+            throw;
         }
 
         throw new InvalidOperationException();

VoidCat/VoidCat.csproj

@@ -16,11 +16,16 @@
 		<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.1" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.1" />
 		<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.14.0" />
+		<PackageReference Include="MSBuildGitHash" Version="2.0.2">
+		  <PrivateAssets>all</PrivateAssets>
+		  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+		</PackageReference>
 		<PackageReference Include="NBitcoin" Version="6.0.19" />
 		<PackageReference Include="nClam" Version="7.0.0" />
 		<PackageReference Include="prometheus-net.AspNetCore" Version="5.0.2" />
 		<PackageReference Include="Seq.Extensions.Logging" Version="6.0.0" />
 		<PackageReference Include="StackExchange.Redis" Version="2.5.27-prerelease" />
+		<PackageReference Include="Swashbuckle.AspNetCore" Version="6.2.3" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 	</ItemGroup>
 	<ItemGroup>
@@ -64,4 +69,17 @@
 		</ItemGroup>
 		<Copy SourceFiles="@(_CopyItems)" DestinationFolder="$(PublishDir)wwwroot\%(RecursiveDir)" />
 	</Target>
+	<ItemGroup>
+		<AssemblyMetadata Include="%(AssemblyMetadataCommand.Identity)">
+			<Value>$(Value)</Value>
+		</AssemblyMetadata>
+		<AssemblyAttributes Include="AssemblyMetadata">
+			<_Parameter1>BuildTime</_Parameter1>
+			<_Parameter2>$([System.DateTime]::UtcNow.ToBinary())</_Parameter2>
+		</AssemblyAttributes>
+		<Compile Include="$(IntermediateOutputPath)\build_date.cs" />
+	</ItemGroup>
+	<Target Name="GetBuildDate" BeforeTargets="CoreCompile">
+		<WriteCodeFragment Language="C#" OutputFile="$(IntermediateOutputPath)\build_date.cs" AssemblyAttributes="@(AssemblyAttributes)" />
+	</Target>
 </Project>