Add ai-model component for text to 3D model generation

aframevr · Mar 21, 2024 · ccea19c · ccea19c
1 parent bb1e7a4
commit ccea19c
Show file tree

Hide file tree

Showing 9 changed files with 101 additions and 3 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -68,6 +68,13 @@
  "ecmaVersion": 8
  }
  },
+ {
+ /* This module uses ES8 async / await due to WebXR Anchor Module integration */
+ "files": ["./src/components/ai-model.js"],
+ "parserOptions": {
+ "ecmaVersion": 8
+ }
+ },
  {
  /* This module uses ES6 for…of loops, and so is parsed as ES6 to avoid errors. */
  "files": ["./src/components/hand-tracking-controls.js"],

diff --git a/examples/showcase/ai-model/index.html b/examples/showcase/ai-model/index.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="utf-8">
+ <title>Anchor (Mixed Reality) • A-Frame</title>
+ <meta name="description" content="Anchor (Mixed Reality) • A-Frame">
+ <script src="../../../dist/aframe-master.js"></script>
+ <script src="../../js/info-message.js"></script>
+ </head>
+ <body>
+ <a-scene
+ info-message="htmlSrc: #messageText">
+ <a-assets>
+ <a-asset-item id="messageText" src="message.html"></a-asset-item>
+ </a-assets>
+ <a-entity ai-model></a-entity>
+ </a-scene>
+ </body>
+</html>
+
diff --git a/examples/showcase/ai-model/message.html b/examples/showcase/ai-model/message.html
@@ -0,0 +1,3 @@
+<p>
+This demo illustrates how to integrate text to 3D to A-Frame</a>
+</p>
diff --git a/package.json b/package.json
@@ -38,6 +38,7 @@
  "buffer": "^6.0.3",
  "debug": "ngokevin/debug#noTimestamp",
  "deep-assign": "^2.0.0",
+ "@gradio/client": "0.14.0",
  "load-bmfont": "^1.2.3",
  "super-animejs": "^3.1.0",
  "super-three": "0.162.0",
@@ -73,6 +74,7 @@
  "karma-webpack": "^5.0.0",
  "markserv": "github:sukima/markserv#feature/fix-broken-websoketio-link",
  "mocha": "^10.0.0",
+ "node-polyfill-webpack-plugin": "^3.0.0",
  "replace-in-file": "^2.5.3",
  "shelljs": "^0.7.7",
  "shx": "^0.2.2",
@@ -115,7 +117,7 @@
  "webxr"
  ],
  "engines": {
- "node": ">= 4.6.0",
- "npm": ">= 2.15.9"
+ "node": ">= 21.7.1",
+ "npm": ">= 10.5.0"
  }
 }
diff --git a/src/components/ai-model.js b/src/components/ai-model.js
@@ -0,0 +1,35 @@
+var registerComponent = require('../core/component').registerComponent;
+var GRADIOCLIENT = require('../lib/gradio');
+
+/**
+ * AI model loader.
+ */
+module.exports.Component = registerComponent('ai-model', {
+ schema: {
+ prompt: {type: 'string', default: 'robot'},
+ api: {type: 'string', oneOf: ['perflow']},
+ key: {type: 'string', default: ''}
+ },
+
+ init: function () {
+ this.generateAndRender(this.data.prompt);
+ },
+
+ update: function () {
+
+ },
+
+ generateAndRender: async function generateAndRender (prompt) {
+ var modelEl = document.createElement('a-entity');
+ modelEl.setAttribute('position', '0 1.6 -2');
+ modelEl.setAttribute('rotation', '0 180 0');
+ var app = await GRADIOCLIENT('hansyan/perflow-triposr', {});
+ var result = await app.predict(
+ '/render',
+ (await app.predict('/generate', [prompt, '0'])).data
+ );
+ console.log('Model URL: ' + result.data[0].url);
+ modelEl.setAttribute('obj-model', {obj: result.data[0].url});
+ this.el.sceneEl.appendChild(modelEl);
+ }
+});
diff --git a/src/components/index.js b/src/components/index.js
@@ -1,3 +1,4 @@
+require('./ai-model');
 require('./animation');
 require('./anchored');
 require('./camera');

diff --git a/src/lib/gradio.js b/src/lib/gradio.js
@@ -0,0 +1,3 @@
+var GRADIOCLIENT = require('./gradio.module.js').default;
+
+module.exports = GRADIOCLIENT;
diff --git a/src/lib/gradio.module.js b/src/lib/gradio.module.js
@@ -0,0 +1,5 @@
+import { client } from '@gradio/client';
+
+var GRADIOCLIENT = client;
+
+export default GRADIOCLIENT;
diff --git a/webpack.config.js b/webpack.config.js
@@ -1,5 +1,6 @@
 var path = require('path');
 var webpack = require('webpack');
+var NodePolyfillPlugin = require('node-polyfill-webpack-plugin');
 
 module.exports = {
  entry: './src/index.js',
@@ -29,11 +30,32 @@ module.exports = {
  new webpack.ProvidePlugin({
  process: 'process/browser',
  Buffer: ['buffer', 'Buffer']
- })
+ }),
+ new webpack.NormalModuleReplacementPlugin(/node:/, (resource) => {
+ resource.request = resource.request.replace(/^node:/, '');
+ }),
+ new NodePolyfillPlugin()
  ],
  resolve: {
  alias: {
  three: 'super-three'
+ },
+ fallback: {
+ // fixes proxy-agent dependencies
+ crypto: false,
+ net: false,
+ dns: false,
+ tls: false,
+ assert: false,
+ http: false,
+ https: false,
+ // fixes next-i18next dependencies
+ path: false,
+ fs: false,
+ // fixes mapbox dependencies
+ events: false,
+ // fixes sentry dependencies
+ process: false
  }
  },
  module: {