Add support for Google buckets storage (#233)

BaGet.sln

@@ -40,6 +40,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.Database.MySql", "src
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaGet.Database.PostgreSql", "src\BaGet.Database.PostgreSql\BaGet.Database.PostgreSql.csproj", "{F48F201A-4DEE-4D5B-9C0B-59490FE942FA}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BaGet.GCP", "src\BaGet.GCP\BaGet.GCP.csproj", "{D7D60BA0-FF7F-4B37-815C-74D487C5176E}"
+EndProject
 Global
  GlobalSection(SolutionConfigurationPlatforms) = preSolution
  Debug|Any CPU = Debug|Any CPU
@@ -102,6 +104,10 @@ Global
  {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
  {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
  {F48F201A-4DEE-4D5B-9C0B-59490FE942FA}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D7D60BA0-FF7F-4B37-815C-74D487C5176E}.Release|Any CPU.Build.0 = Release|Any CPU
  EndGlobalSection
  GlobalSection(SolutionProperties) = preSolution
  HideSolutionNode = FALSE
@@ -121,6 +127,7 @@ Global
  {4C513AFC-BA7B-471D-B8F6-268E7AD2074C} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
  {A4375529-E855-4D46-AA4F-B3FE630C3DE1} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
  {F48F201A-4DEE-4D5B-9C0B-59490FE942FA} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
+ {D7D60BA0-FF7F-4B37-815C-74D487C5176E} = {26A0B557-53FB-4B9A-94C4-BCCF1BDCB0CC}
  EndGlobalSection
  GlobalSection(ExtensibilityGlobals) = postSolution
  SolutionGuid = {1423C027-2C90-417F-8629-2A4CF107C055}

docs/cloud/gcp.md

@@ -1,9 +1,44 @@
-# Running BaGet on the Google Cloud
+# Running BaGet on Google Cloud
 
 !!! warning
  This page is a work in progress!
 
-Sadly, BaGet does not support GCP today. We're open source and accept contributions!
+We're open source and accept contributions!
 [Fork us on GitHub](https://github.com/loic-sharma/BaGet).
 
-For now, please refer to the [Azure documentation](azure).
+## Google Cloud Storage
+
+Packages can be stored in [Google Cloud Storage](https://cloud.google.com/storage/).
+
+### Setup
+
+Follow the instructions in [Using Cloud Storage](https://cloud.google.com/appengine/docs/flexible/dotnet/using-cloud-storage) to:
+
+* Create a bucket
+* Set up a service account and download credentials
+* Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON file you downloaded
+
+### Configuration
+
+Configure BaGet to use GCS by updating the [`appsettings.json`](https://github.com/loic-sharma/BaGet/blob/master/src/BaGet/appsettings.json) file:
+
+```json
+{
+ ...
+
+ "Storage": {
+ "Type": "GoogleCloud",
+ "BucketName": "your-gcs-bucket"
+ },
+
+ ...
+}
+```
+
+## Google Cloud SQL
+
+* TODO
+
+## Google AppEngine
+
+* TODO

src/BaGet.AWS/S3StorageService.cs

@@ -17,11 +17,14 @@ public class S3StorageService : IStorageService
  private readonly string _prefix;
  private readonly AmazonS3Client _client;
 
- public S3StorageService(IOptions<S3StorageOptions> options, AmazonS3Client client)
+ public S3StorageService(IOptionsSnapshot<S3StorageOptions> options, AmazonS3Client client)
  {
+ if (options == null)
+ throw new ArgumentNullException(nameof(options));
+
  _bucket = options.Value.Bucket;
  _prefix = options.Value.Prefix;
- _client = client;
+ _client = client ?? throw new ArgumentNullException(nameof(client));
 
  if (!string.IsNullOrEmpty(_prefix) && !_prefix.EndsWith(Separator))
  _prefix += Separator;

src/BaGet.Core/Configuration/StorageOptions.cs

@@ -9,6 +9,7 @@ public enum StorageType
  {
  FileSystem = 0,
  AzureBlobStorage = 1,
- AwsS3 = 2
+ AwsS3 = 2,
+ GoogleCloud = 3,
  }
 }

src/BaGet.GCP/BaGet.GCP.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+ <PropertyGroup>
+ <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+ </PropertyGroup>
+
+ <ItemGroup>
+ <PackageReference Include="Google.Cloud.Storage.V1" Version="2.2.1" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ProjectReference Include="..\BaGet.Core\BaGet.Core.csproj" />
+ </ItemGroup>
+
+</Project>

src/BaGet.GCP/Configuration/GoogleCloudStorageOptions.cs

@@ -0,0 +1,11 @@
+using System.ComponentModel.DataAnnotations;
+using BaGet.Core.Configuration;
+
+namespace BaGet.GCP.Configuration
+{
+ public class GoogleCloudStorageOptions : StorageOptions
+ {
+ [Required]
+ public string BucketName { get; set; }
+ }
+}

src/BaGet.GCP/Extensions/ServiceCollectionExtensions.cs

@@ -0,0 +1,15 @@
+using BaGet.Core.Services;
+using BaGet.GCP.Services;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace BaGet.GCP.Extensions
+{
+ public static class ServiceCollectionExtensions
+ {
+ public static IServiceCollection AddGoogleCloudStorageService(this IServiceCollection services)
+ {
+ services.AddTransient<GoogleCloudStorageService>();
+ return services;
+ }
+ }
+}

src/BaGet.GCP/Services/GoogleCloudStorageService.cs

@@ -0,0 +1,101 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using BaGet.Core.Services;
+using BaGet.GCP.Configuration;
+using Google;
+using Google.Cloud.Storage.V1;
+using Microsoft.Extensions.Options;
+
+namespace BaGet.GCP.Services
+{
+ public class GoogleCloudStorageService : IStorageService
+ {
+ private readonly string _bucketName;
+
+ public GoogleCloudStorageService(IOptionsSnapshot<GoogleCloudStorageOptions> options)
+ {
+ if (options == null)
+ throw new ArgumentNullException(nameof(options));
+
+ _bucketName = options.Value.BucketName;
+ }
+
+ public async Task<Stream> GetAsync(string path, CancellationToken cancellationToken = default)
+ {
+ using (var storage = await StorageClient.CreateAsync())
+ {
+ var stream = new MemoryStream();
+ await storage.DownloadObjectAsync(_bucketName, CoercePath(path), stream, cancellationToken: cancellationToken);
+ stream.Position = 0;
+ return stream;
+ }
+ }
+
+ public Task<Uri> GetDownloadUriAsync(string path, CancellationToken cancellationToken = default)
+ {
+ // returns an Authenticated Browser Download URL: https://cloud.google.com/storage/docs/request-endpoints#cookieauth
+ return Task.FromResult(new Uri($"https://storage.googleapis.com/{_bucketName}/{CoercePath(path).TrimStart('/')}"));
+ }
+
+ public async Task<PutResult> PutAsync(string path, Stream content, string contentType, CancellationToken cancellationToken = default)
+ {
+ using (var storage = await StorageClient.CreateAsync())
+ using (var seekableContent = new MemoryStream())
+ {
+ await content.CopyToAsync(seekableContent, 65536, cancellationToken);
+ seekableContent.Position = 0;
+
+ var objectName = CoercePath(path);
+
+ try
+ {
+ // attempt to upload, succeeding only if the object doesn't exist
+ await storage.UploadObjectAsync(_bucketName, objectName, contentType, seekableContent, new UploadObjectOptions { IfGenerationMatch = 0 }, cancellationToken);
+ return PutResult.Success;
+ }
+ catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.PreconditionFailed)
+ {
+ // the object already exists; get the hash of its content from its metadata
+ var existingObject = await storage.GetObjectAsync(_bucketName, objectName, cancellationToken: cancellationToken);
+ var existingHash = Convert.FromBase64String(existingObject.Md5Hash);
+
+ // hash the content that was uploaded
+ seekableContent.Position = 0;
+ byte[] contentHash;
+ using (var md5 = MD5.Create())
+ contentHash = md5.ComputeHash(seekableContent);
+
+ // conflict if the two hashes are different
+ return existingHash.SequenceEqual(contentHash) ? PutResult.AlreadyExists : PutResult.Conflict;
+ }
+ }
+ }
+
+ public async Task DeleteAsync(string path, CancellationToken cancellationToken = default)
+ {
+ using (var storage = await StorageClient.CreateAsync())
+ {
+ try
+ {
+ var obj = await storage.GetObjectAsync(_bucketName, CoercePath(path), cancellationToken: cancellationToken);
+ await storage.DeleteObjectAsync(obj, cancellationToken: cancellationToken);
+ }
+ catch (GoogleApiException e) when (e.HttpStatusCode == HttpStatusCode.NotFound)
+ {
+ }
+ }
+ }
+
+ private static string CoercePath(string path)
+ {
+ // although Google Cloud Storage objects exist in a flat namespace, using forward slashes allows the objects to
+ // be exposed as nested subdirectories, e.g., when browsing via Google Cloud Console
+ return path.Replace('\\', '/');
+ }
+ }
+}

src/BaGet/BaGet.csproj

@@ -31,6 +31,7 @@
  <ProjectReference Include="..\BaGet.Database.Sqlite\BaGet.Database.Sqlite.csproj" />
  <ProjectReference Include="..\BaGet.Database.SqlServer\BaGet.Database.SqlServer.csproj" />
  <ProjectReference Include="..\BaGet.Database.PostgreSql\BaGet.Database.PostgreSql.csproj" />
+ <ProjectReference Include="..\BaGet.GCP\BaGet.GCP.csproj" />
  <ProjectReference Include="..\BaGet.Protocol\BaGet.Protocol.csproj" />
  </ItemGroup>
 

src/BaGet/Extensions/IServiceCollectionExtensions.cs

@@ -18,6 +18,9 @@
 using BaGet.Database.PostgreSql;
 using BaGet.Database.Sqlite;
 using BaGet.Database.SqlServer;
+using BaGet.GCP.Configuration;
+using BaGet.GCP.Extensions;
+using BaGet.GCP.Services;
 using BaGet.Protocol;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
@@ -44,6 +47,7 @@ public static class IServiceCollectionExtensions
 
  services.ConfigureAzure(configuration);
  services.ConfigureAws(configuration);
+ services.ConfigureGcp(configuration);
 
  if (httpServices)
  {
@@ -144,6 +148,15 @@ public static IServiceCollection AddBaGetContext(this IServiceCollection service
  return services;
  }
 
+ public static IServiceCollection ConfigureGcp(
+ this IServiceCollection services,
+ IConfiguration configuration)
+ {
+ services.ConfigureAndValidate<GoogleCloudStorageOptions>(configuration.GetSection(nameof(BaGetOptions.Storage)));
+
+ return services;
+ }
+
  public static IServiceCollection AddStorageProviders(this IServiceCollection services)
  {
  services.AddTransient<FileStorageService>();
@@ -152,6 +165,7 @@ public static IServiceCollection AddStorageProviders(this IServiceCollection ser
 
  services.AddBlobStorageService();
  services.AddS3StorageService();
+ services.AddGoogleCloudStorageService();
 
  services.AddTransient<IStorageService>(provider =>
  {
@@ -168,6 +182,9 @@ public static IServiceCollection AddStorageProviders(this IServiceCollection ser
  case StorageType.AwsS3:
  return provider.GetRequiredService<S3StorageService>();
 
+ case StorageType.GoogleCloud:
+ return provider.GetRequiredService<GoogleCloudStorageService>();
+
  default:
  throw new InvalidOperationException(
  $"Unsupported storage service: {options.Value.Storage.Type}");