feat(webserver, db): refactor gitlab and github integrations into unified integration / third party repository services

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "crates/llama-cpp-bindings/llama.cpp"]
+[submodule "crates/llama-cpp-server/llama.cpp"]
  path = crates/llama-cpp-server/llama.cpp
  url = https://github.com/ggerganov/llama.cpp

ee/tabby-db-macros/src/lib.rs

@@ -109,7 +109,7 @@ pub fn query_paged_as(input: TokenStream) -> TokenStream {
  let backwards = input.backwards;
  quote! {
  sqlx::query_as(&crate::make_pagination_query_with_condition({
- let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM " + #table_name);
+ let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM (SELECT * FROM " + #table_name + ")");
  &#table_name
  }, &[ #(#column_args),* ], #limit, #skip_id, #backwards, #condition))
  }

ee/tabby-db/migrations/0029_merged-provider-tables.up.sql

@@ -2,7 +2,7 @@ CREATE TABLE integration_access_tokens(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
  kind TEXT NOT NULL,
  display_name TEXT NOT NULL,
- access_token TEXT,
+ access_token TEXT NOT NULL,
  error TEXT,
  created_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
  updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
@@ -20,3 +20,9 @@ CREATE TABLE provided_repositories(
  FOREIGN KEY (integration_access_token_id) REFERENCES integration_access_tokens(id) ON DELETE CASCADE,
  CONSTRAINT idx_unique_provider_id_vendor_id UNIQUE (integration_access_token_id, vendor_id)
 );
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+ SELECT 'github', display_name, access_token FROM github_repository_provider;
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+ SELECT 'gitlab', display_name, access_token FROM gitlab_repository_provider;

ee/tabby-db/schema/schema.sql

@@ -176,7 +176,7 @@ CREATE TABLE integration_access_tokens(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
  kind TEXT NOT NULL,
  display_name TEXT NOT NULL,
- access_token TEXT,
+ access_token TEXT NOT NULL,
  error TEXT,
  created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
  updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))

ee/tabby-db/src/integration_access_tokens.rs

@@ -10,17 +10,17 @@
  pub kind: String,
  pub error: Option<String>,
  pub display_name: String,
- pub access_token: Option<String>,
+ pub access_token: String,
  pub created_at: DateTimeUtc,
  pub updated_at: DateTimeUtc,
 }
 
 impl DbConn {
  pub async fn create_integration_access_token(
  &self,
- kind: &str,
- name: &str,
- access_token: &str,
+ kind: String,
+ name: String,
+ access_token: String,
  ) -> Result<i64> {
  let res = query!(
  "INSERT INTO integration_access_tokens(kind, display_name, access_token) VALUES (?, ?, ?);",
@@ -69,7 +69,7 @@
  access_token: Option<String>,
  ) -> Result<()> {
  let access_token = match access_token {
- Some(access_token) => Some(access_token),
+ Some(access_token) => access_token,
  None => self.get_integration_access_token(id).await?.access_token,
  };
 
@@ -126,7 +126,7 @@
  });
  conditions.extend(id_condition);
 
- let kind_condition = kind.map(|kind| format!("kind = {kind}"));
+ let kind_condition = kind.map(|kind| format!("kind = '{kind}'"));
  conditions.extend(kind_condition);
 
  let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));

ee/tabby-db/src/lib.rs

@@ -12,9 +12,11 @@ use chrono::{DateTime, Duration, NaiveDateTime, Utc};
 pub use email_setting::EmailSettingDAO;
 pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO};
 pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO};
+pub use integration_access_tokens::IntegrationAccessTokenDAO;
 pub use invitations::InvitationDAO;
 pub use job_runs::JobRunDAO;
 pub use oauth_credential::OAuthCredentialDAO;
+pub use provided_repositories::ProvidedRepositoryDAO;
 pub use repositories::RepositoryDAO;
 pub use server_setting::ServerSettingDAO;
 use sqlx::{

ee/tabby-db/src/provided_repositories.rs

@@ -61,7 +61,7 @@
 
  pub async fn list_provided_repositories(
  &self,
- provider_ids: Vec<i64>,
+ integration_ids: Vec<i64>,
  kind: Option<String>,
  active: Option<bool>,
  limit: Option<usize>,
@@ -70,35 +70,37 @@
  ) -> Result<Vec<ProvidedRepositoryDAO>> {
  let mut conditions = vec![];
 
- let provider_ids = provider_ids
+ let integration_ids = integration_ids
  .into_iter()
  .map(|id| id.to_string())
  .collect::<Vec<_>>()
  .join(", ");
- if !provider_ids.is_empty() {
- conditions.push(format!("access_token_provider_id IN ({provider_ids})"));
+ if !integration_ids.is_empty() {
+ conditions.push(format!(
+ "integration_access_token_id IN ({integration_ids})"
+ ));
  }
 
  let active_filter = active.map(|active| format!("active = {active}"));
  conditions.extend(active_filter);
 
- let kind_filter = kind.map(|kind| format!("kind = {kind}"));
+ let kind_filter = kind.map(|kind| format!("kind = '{kind}'"));
  conditions.extend(kind_filter);
 
  let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
 
  let repos = query_paged_as!(
  ProvidedRepositoryDAO,
- "provided_repositories",
+ "provided_repositories JOIN integration_access_tokens ON integration_access_token_id = integration_access_tokens.id",
  [
  "id",
  "vendor_id",
  "name",
  "git_url",
  "active",
  "integration_access_token_id",
- "created_at" as "created_at: DateTimeUtc",
- "updated_at" as "updated_at: DateTimeUtc"
+ "created_at",
+ "updated_at"
  ],
  limit,
  skip_id,
@@ -128,3 +130,16 @@
  Ok(())
  }
 }
+
+#[cfg(test)]
+mod tests {
+ use crate::DbConn;
+
+ #[tokio::test]
+ async fn test_list_provided_repositories() {
+ let db = DbConn::new_in_memory().await.unwrap();
+ db.list_github_provided_repositories(vec![], None, None, None, false)
+ .await
+ .unwrap();
+ }
+}

ee/tabby-schema/graphql/schema.graphql

@@ -376,9 +376,9 @@ type Query {
  users(after: String, before: String, first: Int, last: Int): UserConnection!
  invitations(after: String, before: String, first: Int, last: Int): InvitationConnection!
  githubRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GithubRepositoryProviderConnection!
- githubRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
+ githubRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GithubProvidedRepositoryConnection!
  gitlabRepositoryProviders(ids: [ID!], after: String, before: String, first: Int, last: Int): GitlabRepositoryProviderConnection!
- gitlabRepositories(providerIds: [ID!]!, active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
+ gitlabRepositories(providerIds: [ID!], active: Boolean, after: String, before: String, first: Int, last: Int): GitlabProvidedRepositoryConnection!
  jobRuns(ids: [ID!], jobs: [String!], after: String, before: String, first: Int, last: Int): JobRunConnection!
  jobRunStats(jobs: [String!]): JobStats!
  emailSetting: EmailSetting

ee/tabby-schema/src/dao.rs

@@ -3,21 +3,26 @@
 use lazy_static::lazy_static;
 use tabby_db::{
  EmailSettingDAO, GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO,
- GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, InvitationDAO, JobRunDAO,
- OAuthCredentialDAO, RepositoryDAO, ServerSettingDAO, UserDAO, UserEventDAO,
+ GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, IntegrationAccessTokenDAO,
+ InvitationDAO, JobRunDAO, OAuthCredentialDAO, ProvidedRepositoryDAO, RepositoryDAO,
+ ServerSettingDAO, UserDAO, UserEventDAO,
 };
 
-use crate::schema::{
- auth::{self, OAuthCredential, OAuthProvider},
- email::{AuthMethod, EmailSetting, Encryption},
- job,
- repository::{
- GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
- GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
+use crate::{
+ integration::{IntegrationAccessToken, IntegrationKind, IntegrationStatus},
+ repository::ProvidedRepository,
+ schema::{
+ auth::{self, OAuthCredential, OAuthProvider},
+ email::{AuthMethod, EmailSetting, Encryption},
+ job,
+ repository::{
+ GitRepository, GithubProvidedRepository, GithubRepositoryProvider,
+ GitlabProvidedRepository, GitlabRepositoryProvider, RepositoryProviderStatus,
+ },
+ setting::{NetworkSetting, SecuritySetting},
+ user_event::{EventKind, UserEvent},
+ CoreError,
  },
- setting::{NetworkSetting, SecuritySetting},
- user_event::{EventKind, UserEvent},
- CoreError,
 };
 
 impl From<InvitationDAO> for auth::Invitation {
@@ -125,6 +130,44 @@
  }
 }
 
+impl TryFrom<ProvidedRepositoryDAO> for ProvidedRepository {
+ type Error = anyhow::Error;
+ fn try_from(value: ProvidedRepositoryDAO) -> Result<Self, Self::Error> {
+ Ok(Self {
+ id: value.id.as_id(),
+ integration_id: value.integration_access_token_id.as_id(),
+ active: value.active,
+ display_name: value.name,
+ git_url: value.git_url,
+ vendor_id: value.vendor_id,
+ created_at: *value.created_at,
+ updated_at: *value.updated_at,
+ })
+ }
+}
+
+impl TryFrom<IntegrationAccessTokenDAO> for IntegrationAccessToken {
+ type Error = anyhow::Error;
+ fn try_from(value: IntegrationAccessTokenDAO) -> anyhow::Result<Self> {
+ let status = if *value.created_at == *value.updated_at {
+ IntegrationStatus::Pending
+ } else if value.error.is_some() {
+ IntegrationStatus::Failed
+ } else {
+ IntegrationStatus::Ready
+ };
+ Ok(Self {
+ id: value.id.as_id(),
+ kind: IntegrationKind::from_enum_str(&value.kind)?,
+ display_name: value.display_name,
+ access_token: value.access_token,
+ created_at: *value.created_at,
+ updated_at: *value.updated_at,
+ status,
+ })
+ }
+}
+
 impl From<GithubRepositoryProviderDAO> for GithubRepositoryProvider {
  fn from(value: GithubRepositoryProviderDAO) -> Self {
  Self {
@@ -139,6 +182,64 @@
  }
 }
 
+impl From<ProvidedRepository> for GithubProvidedRepository {
+ fn from(value: ProvidedRepository) -> Self {
+ Self {
+ id: value.id,
+ vendor_id: value.vendor_id,
+ github_repository_provider_id: value.integration_id,
+ name: value.display_name,
+ git_url: value.git_url,
+ active: value.active,
+ }
+ }
+}
+
+impl From<IntegrationAccessToken> for GithubRepositoryProvider {
+ fn from(value: IntegrationAccessToken) -> Self {
+ Self {
+ id: value.id,
+ display_name: value.display_name,
+ status: value.status.into(),
+ access_token: Some(value.access_token),
+ }
+ }
+}
+
+impl From<IntegrationAccessToken> for GitlabRepositoryProvider {
+ fn from(value: IntegrationAccessToken) -> Self {
+ Self {
+ id: value.id,
+ display_name: value.display_name,
+ status: value.status.into(),
+ access_token: Some(value.access_token),
+ }
+ }
+}
+
+impl From<IntegrationStatus> for RepositoryProviderStatus {
+ fn from(value: IntegrationStatus) -> Self {
+ match value {
+ IntegrationStatus::Ready => Self::Ready,
+ IntegrationStatus::Pending => Self::Pending,
+ IntegrationStatus::Failed => Self::Failed,
+ }
+ }
+}
+
+impl From<ProvidedRepository> for GitlabProvidedRepository {
+ fn from(value: ProvidedRepository) -> Self {
+ Self {
+ id: value.id,
+ vendor_id: value.vendor_id,
+ gitlab_repository_provider_id: value.integration_id,
+ name: value.display_name,
+ git_url: value.git_url,
+ active: value.active,
+ }
+ }
+}
+
 impl From<GithubProvidedRepositoryDAO> for GithubProvidedRepository {
  fn from(value: GithubProvidedRepositoryDAO) -> Self {
  Self {
@@ -255,6 +356,23 @@
  }
 }
 
+impl DbEnum for IntegrationKind {
+ fn as_enum_str(&self) -> &'static str {
+ match self {
+ IntegrationKind::Github => "github",
+ IntegrationKind::Gitlab => "gitlab",
+ }
+ }
+
+ fn from_enum_str(s: &str) -> anyhow::Result<Self> {
+ match s {
+ "github" => Ok(IntegrationKind::Github),
+ "gitlab" => Ok(IntegrationKind::Gitlab),
+ _ => bail!("{s} is not a valid value for ProviderKind"),
+ }
+ }
+}
+
 impl DbEnum for Encryption {
  fn as_enum_str(&self) -> &'static str {
  match self {