Add ai-model to for text to 3D model generation

aframevr · Mar 21, 2024 · eb473e0 · eb473e0
1 parent bb1e7a4
commit eb473e0
Show file tree

Hide file tree

Showing 8 changed files with 102 additions and 3 deletions.
diff --git a/examples/showcase/ai-model/index.html b/examples/showcase/ai-model/index.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="utf-8">
+ <title>Anchor (Mixed Reality) • A-Frame</title>
+ <meta name="description" content="Anchor (Mixed Reality) • A-Frame">
+ <script src="../../../dist/aframe-master.js"></script>
+ <script src="../../js/info-message.js"></script>
+ </head>
+ <body>
+ <a-scene
+ info-message="htmlSrc: #messageText">
+ <a-assets>
+ <a-asset-item id="messageText" src="message.html"></a-asset-item>
+ </a-assets>
+ <a-entity ai-model></a-entity>
+ </a-scene>
+ </body>
+</html>
+
diff --git a/examples/showcase/ai-model/message.html b/examples/showcase/ai-model/message.html
@@ -0,0 +1,11 @@
+<p>
+This demo requires a browser supporting the <a href="https://immersive-web.github.io/anchors/">WebXR Anchors Module</a>
+</p>
+
+<p>
+In AR mode the user can pinch to grab the painting and release it at any position and orientation. The painting will remain anchored to the real world coordinate where it was released by the user. The anchor will persists upon page reload.
+</p>
+
+<p>
+Paintings by <a href="https://en.wikipedia.org/wiki/Vincent_van_Gogh">Van Gogh</a>, <a href="https://en.wikipedia.org/wiki/Alphonse_Mucha">Alphonse Mucha</a>, <a href="https://en.wikipedia.org/wiki/El_Greco">El Greco</a> and <a href="https://en.wikipedia.org/wiki/Edvard_Munch">Edvard Munch</a>. Frame model by <a href="https://sketchfab.com/3d-models/vintage-painting-dani-3fdd92904c2b44028bef28b33e897d9f">theGentleGiant</a>
+</p>
diff --git a/package.json b/package.json
@@ -38,6 +38,7 @@
  "buffer": "^6.0.3",
  "debug": "ngokevin/debug#noTimestamp",
  "deep-assign": "^2.0.0",
+ "@gradio/client": "0.14.0",
  "load-bmfont": "^1.2.3",
  "super-animejs": "^3.1.0",
  "super-three": "0.162.0",
@@ -73,6 +74,7 @@
  "karma-webpack": "^5.0.0",
  "markserv": "github:sukima/markserv#feature/fix-broken-websoketio-link",
  "mocha": "^10.0.0",
+ "node-polyfill-webpack-plugin": "^3.0.0",
  "replace-in-file": "^2.5.3",
  "shelljs": "^0.7.7",
  "shx": "^0.2.2",
@@ -115,7 +117,7 @@
  "webxr"
  ],
  "engines": {
- "node": ">= 4.6.0",
- "npm": ">= 2.15.9"
+ "node": ">= 21.7.1",
+ "npm": ">= 10.5.0"
  }
 }
diff --git a/src/components/ai-model.js b/src/components/ai-model.js
@@ -0,0 +1,35 @@
+var registerComponent = require('../core/component').registerComponent;
+var GRADIOCLIENT = require('../lib/gradio');
+
+/**
+ * AI model loader.
+ */
+module.exports.Component = registerComponent('ai-model', {
+ schema: {
+ prompt: {type: 'string', default: 'robot'},
+ api: {type: 'string', oneOf: ['perflow']},
+ key: {type: 'string', default: ''}
+ },
+
+ init: function () {
+ this.generateAndRender(this.data.prompt);
+ },
+
+ update: function () {
+
+ },
+
+ generateAndRender: async function generateAndRender (prompt) {
+ var modelEl = document.createElement('a-entity');
+ modelEl.setAttribute('position', '0 1.6 -2');
+ modelEl.setAttribute('rotation', '0 180 0');
+ var app = await GRADIOCLIENT('hansyan/perflow-triposr', {});
+ var result = await app.predict(
+ '/render',
+ (await app.predict('/generate', [prompt, '0'])).data
+ );
+ console.log('Model URL: ' + result.data[0].url);
+ modelEl.setAttribute('obj-model', {obj: result.data[0].url});
+ this.el.sceneEl.appendChild(modelEl);
+ }
+});
diff --git a/src/components/index.js b/src/components/index.js
@@ -1,3 +1,4 @@
+require('./ai-model');
 require('./animation');
 require('./anchored');
 require('./camera');

diff --git a/src/lib/gradio.js b/src/lib/gradio.js
@@ -0,0 +1,3 @@
+var GRADIOCLIENT = require('./gradio.module.js').default;
+
+module.exports = GRADIOCLIENT;
diff --git a/src/lib/gradio.module.js b/src/lib/gradio.module.js
@@ -0,0 +1,5 @@
+import { client } from '@gradio/client';
+
+var GRADIOCLIENT = client;
+
+export default GRADIOCLIENT;
diff --git a/webpack.config.js b/webpack.config.js
@@ -1,5 +1,6 @@
 var path = require('path');
 var webpack = require('webpack');
+var NodePolyfillPlugin = require('node-polyfill-webpack-plugin');
 
 module.exports = {
  entry: './src/index.js',
@@ -29,11 +30,32 @@ module.exports = {
  new webpack.ProvidePlugin({
  process: 'process/browser',
  Buffer: ['buffer', 'Buffer']
- })
+ }),
+ new webpack.NormalModuleReplacementPlugin(/node:/, (resource) => {
+ resource.request = resource.request.replace(/^node:/, '');
+ }),
+ new NodePolyfillPlugin()
  ],
  resolve: {
  alias: {
  three: 'super-three'
+ },
+ fallback: {
+ // fixes proxy-agent dependencies
+ crypto: false,
+ net: false,
+ dns: false,
+ tls: false,
+ assert: false,
+ http: false,
+ https: false,
+ // fixes next-i18next dependencies
+ path: false,
+ fs: false,
+ // fixes mapbox dependencies
+ events: false,
+ // fixes sentry dependencies
+ process: false
  }
  },
  module: {