feat(webserver, db): refactor gitlab and github integrations into uni…

…fied integration / third party repository services (#2088) * draft: begin implementing integration / third party repository service * Progress on refactor * [autofix.ci] apply automated fixes * Use join instead of adding kind as a field of repositories * Update schema * [autofix.ci] apply automated fixes * Implement sync_repositories * Make access_token field required * [autofix.ci] apply automated fixes * Implement IntegrationService * Implement service layer * [autofix.ci] apply automated fixes * Switch graphQL endpoints to use new service * [autofix.ci] apply automated fixes * Fix queries * Fully working * Rebase * Update submodules * Cleanup imports * [autofix.ci] apply automated fixes * Apply suggestions * [autofix.ci] apply automated fixes * Add back updated_at --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
TabbyML · May 17, 2024 · 39175ae · 39175ae
1 parent 8ab9b56
commit 39175ae
Show file tree

Hide file tree

Showing 24 changed files with 1,065 additions and 98 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
-[submodule "crates/llama-cpp-bindings/llama.cpp"]
+[submodule "crates/llama-cpp-server/llama.cpp"]
  path = crates/llama-cpp-server/llama.cpp
  url = https://github.com/ggerganov/llama.cpp
diff --git a/ee/tabby-db-macros/src/lib.rs b/ee/tabby-db-macros/src/lib.rs
@@ -109,7 +109,7 @@ pub fn query_paged_as(input: TokenStream) -> TokenStream {
  let backwards = input.backwards;
  quote! {
  sqlx::query_as(&crate::make_pagination_query_with_condition({
- let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM " + #table_name);
+ let _ = sqlx::query_as!(#typ, "SELECT " + #columns + " FROM (SELECT * FROM " + #table_name + ")");
  &#table_name
  }, &[ #(#column_args),* ], #limit, #skip_id, #backwards, #condition))
  }

diff --git a/ee/tabby-db/migrations/0029_merged-provider-tables.up.sql b/ee/tabby-db/migrations/0029_merged-provider-tables.up.sql
@@ -2,10 +2,11 @@ CREATE TABLE integration_access_tokens(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
  kind TEXT NOT NULL,
  display_name TEXT NOT NULL,
- access_token TEXT,
+ access_token TEXT NOT NULL,
  error TEXT,
  created_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
- updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
+ updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')),
+ synced_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now'))
 );
 
 CREATE TABLE provided_repositories(
@@ -20,3 +21,9 @@ CREATE TABLE provided_repositories(
  FOREIGN KEY (integration_access_token_id) REFERENCES integration_access_tokens(id) ON DELETE CASCADE,
  CONSTRAINT idx_unique_provider_id_vendor_id UNIQUE (integration_access_token_id, vendor_id)
 );
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+ SELECT 'github', display_name, access_token FROM github_repository_provider WHERE access_token IS NOT NULL;
+
+INSERT INTO integration_access_tokens(kind, display_name, access_token)
+ SELECT 'gitlab', display_name, access_token FROM gitlab_repository_provider WHERE access_token IS NOT NULL;
diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite
diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql
@@ -176,10 +176,11 @@ CREATE TABLE integration_access_tokens(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
  kind TEXT NOT NULL,
  display_name TEXT NOT NULL,
- access_token TEXT,
+ access_token TEXT NOT NULL,
  error TEXT,
  created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
- updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))
+ updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')),
+ synced_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now'))
 );
 CREATE TABLE provided_repositories(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,

diff --git a/ee/tabby-db/schema/schema.svg b/ee/tabby-db/schema/schema.svg
diff --git a/ee/tabby-db/src/integration_access_tokens.rs b/ee/tabby-db/src/integration_access_tokens.rs
@@ -10,17 +10,18 @@ pub struct IntegrationAccessTokenDAO {
  pub kind: String,
  pub error: Option<String>,
  pub display_name: String,
- pub access_token: Option<String>,
+ pub access_token: String,
  pub created_at: DateTimeUtc,
  pub updated_at: DateTimeUtc,
+ pub synced_at: DateTimeUtc,
 }
 
 impl DbConn {
  pub async fn create_integration_access_token(
  &self,
- kind: &str,
- name: &str,
- access_token: &str,
+ kind: String,
+ name: String,
+ access_token: String,
  ) -> Result<i64> {
  let res = query!(
  "INSERT INTO integration_access_tokens(kind, display_name, access_token) VALUES (?, ?, ?);",
@@ -42,8 +43,9 @@ impl DbConn {
  error,
  display_name,
  access_token,
- created_at AS "created_at: DateTimeUtc",
- updated_at AS "updated_at: DateTimeUtc"
+ updated_at,
+ created_at,
+ synced_at
  FROM integration_access_tokens WHERE id = ?;"#,
  id
  )
@@ -52,10 +54,14 @@ impl DbConn {
  Ok(provider)
  }
 
- pub async fn delete_integration_access_token(&self, id: i64) -> Result<()> {
- let res = query!("DELETE FROM integration_access_tokens WHERE id = ?;", id)
- .execute(&self.pool)
- .await?;
+ pub async fn delete_integration_access_token(&self, id: i64, kind: &str) -> Result<()> {
+ let res = query!(
+ "DELETE FROM integration_access_tokens WHERE id = ? AND kind = ?;",
+ id,
+ kind
+ )
+ .execute(&self.pool)
+ .await?;
  if res.rows_affected() != 1 {
  return Err(anyhow!("No integration access token to delete"));
  }
@@ -65,19 +71,21 @@ impl DbConn {
  pub async fn update_integration_access_token(
  &self,
  id: i64,
+ kind: &str,
  display_name: String,
  access_token: Option<String>,
  ) -> Result<()> {
  let access_token = match access_token {
- Some(access_token) => Some(access_token),
+ Some(access_token) => access_token,
  None => self.get_integration_access_token(id).await?.access_token,
  };
 
  let res = query!(
- "UPDATE integration_access_tokens SET display_name = ?, access_token=? WHERE id = ?;",
+ "UPDATE integration_access_tokens SET display_name = ?, access_token = ?, updated_at = DATETIME('now') WHERE id = ? AND kind = ?;",
  display_name,
  access_token,
- id
+ id,
+ kind
  )
  .execute(&self.pool)
  .await?;
@@ -97,7 +105,7 @@ impl DbConn {
  error: Option<String>,
  ) -> Result<()> {
  query!(
- "UPDATE integration_access_tokens SET updated_at = DATETIME('now'), error = ? WHERE id = ?",
+ "UPDATE integration_access_tokens SET synced_at = DATETIME('now'), error = ? WHERE id = ?",
  error,
  id
  )
@@ -126,7 +134,7 @@ impl DbConn {
  });
  conditions.extend(id_condition);
 
- let kind_condition = kind.map(|kind| format!("kind = {kind}"));
+ let kind_condition = kind.map(|kind| format!("kind = '{kind}'"));
  conditions.extend(kind_condition);
 
  let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
@@ -140,8 +148,9 @@ impl DbConn {
  "error",
  "display_name",
  "access_token",
- "created_at" as "created_at: DateTimeUtc",
- "updated_at" as "updated_at: DateTimeUtc"
+ "created_at",
+ "updated_at",
+ "synced_at"
  ],
  limit,
  skip_id,

diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs
@@ -12,9 +12,11 @@ use chrono::{DateTime, Duration, NaiveDateTime, Utc};
 pub use email_setting::EmailSettingDAO;
 pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO};
 pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO};
+pub use integration_access_tokens::IntegrationAccessTokenDAO;
 pub use invitations::InvitationDAO;
 pub use job_runs::JobRunDAO;
 pub use oauth_credential::OAuthCredentialDAO;
+pub use provided_repositories::ProvidedRepositoryDAO;
 pub use repositories::RepositoryDAO;
 pub use server_setting::ServerSettingDAO;
 use sqlx::{

diff --git a/ee/tabby-db/src/provided_repositories.rs b/ee/tabby-db/src/provided_repositories.rs
@@ -61,7 +61,7 @@ impl DbConn {
 
  pub async fn list_provided_repositories(
  &self,
- provider_ids: Vec<i64>,
+ integration_ids: Vec<i64>,
  kind: Option<String>,
  active: Option<bool>,
  limit: Option<usize>,
@@ -70,35 +70,37 @@ impl DbConn {
  ) -> Result<Vec<ProvidedRepositoryDAO>> {
  let mut conditions = vec![];
 
- let provider_ids = provider_ids
+ let integration_ids = integration_ids
  .into_iter()
  .map(|id| id.to_string())
  .collect::<Vec<_>>()
  .join(", ");
- if !provider_ids.is_empty() {
- conditions.push(format!("access_token_provider_id IN ({provider_ids})"));
+ if !integration_ids.is_empty() {
+ conditions.push(format!(
+ "integration_access_token_id IN ({integration_ids})"
+ ));
  }
 
  let active_filter = active.map(|active| format!("active = {active}"));
  conditions.extend(active_filter);
 
- let kind_filter = kind.map(|kind| format!("kind = {kind}"));
+ let kind_filter = kind.map(|kind| format!("kind = '{kind}'"));
  conditions.extend(kind_filter);
 
  let condition = (!conditions.is_empty()).then(|| conditions.join(" AND "));
 
  let repos = query_paged_as!(
  ProvidedRepositoryDAO,
- "provided_repositories",
+ "provided_repositories JOIN integration_access_tokens ON integration_access_token_id = integration_access_tokens.id",
  [
  "id",
  "vendor_id",
  "name",
  "git_url",
  "active",
  "integration_access_token_id",
- "created_at" as "created_at: DateTimeUtc",
- "updated_at" as "updated_at: DateTimeUtc"
+ "created_at",
+ "updated_at"
  ],
  limit,
  skip_id,
@@ -128,3 +130,16 @@ impl DbConn {
  Ok(())
  }
 }
+
+#[cfg(test)]
+mod tests {
+ use crate::DbConn;
+
+ #[tokio::test]
+ async fn test_list_provided_repositories() {
+ let db = DbConn::new_in_memory().await.unwrap();
+ db.list_github_provided_repositories(vec![], None, None, None, false)
+ .await
+ .unwrap();
+ }
+}