Merge pull request #43 from zilliztech/readmeV2

Readme v2

README.md

@@ -1,4 +1,4 @@
-# feder
+# Feder
 
 ## What is feder
 
@@ -13,10 +13,14 @@ Feder is written in **javascript**, and we also provide a python library **feder
 - In IPython environment, it supports users to generate the corresponding visualization directly.
 - In other environments, it supports outputting visualizations as html files, which can be opened by the user through the browser with web service enabled.
 
-### Examples
+### Wiki
 
-- [Javascript example](https://observablehq.com/@min-tian/feder)
-- [Jupternotebook example](https://colab.research.google.com/drive/12L_oJPR-yFDlORpPondsqGNTPVsSsUwi#scrollTo=N3qqBAYxYcbt)
+- [Usage](https://github.com/zilliztech/feder/wiki)
+
+### Online Demos
+
+- [Javascript example (Observable)](https://observablehq.com/@min-tian/feder)
+- [Jupternotebook example (Colab)](https://colab.research.google.com/drive/12L_oJPR-yFDlORpPondsqGNTPVsSsUwi#scrollTo=N3qqBAYxYcbt)
 
 ### HNSW visualization screenshots
 
@@ -28,12 +32,13 @@ Feder is written in **javascript**, and we also provide a python library **feder
 ![image](./fig/ivfflat_fine_polar.png)
 ![image](./fig/ivfflat_fine_project.png)
 
-## Installation
+## Quick Start
+
+### Installation
 
 Use npm or yarn.
 
 ```shell
-#install
 yarn install @zilliz/feder
 ```
 
@@ -58,7 +63,8 @@ const feder = new Feder({
 
 ### Visualize the index structure.
 
-- HNSW - Feder will show the top-3 levels of the hnsw-tree
+- HNSW - Feder will show the top-3 levels of the hnsw-tree.
+- IVF_Flat - Feder will show all the clusters.
 
 ```js
 feder.overview();
@@ -69,48 +75,19 @@ feder.overview();
 Set search parameters (optional) and Specify the query vector.
 
 ```js
-feder.setSearchParams({ k: 8, ef_search: 100 }); // hnsw
-feder.setSearchParams({ k: 8, nprobe: 8 }); // ivf_flat
-feder.search(target_vector);
+feder
+ .setSearchParams({
+ k: 8, // hnsw, ivf_flat
+ ef: 100, // hnsw (ef_search)
+ nprobe: 8, // ivf_flat
+ })
+ .search(target_vector);
 ```
 
-### Advanced
-
-Use **viewParams** to adjust the details of the view.
-
-```js
-const feder = new Feder({
- filePath,
- source: 'hnswlib',
- domSelector,
- viewParams: {
- width: 1000,
- height: 600,
- padding: [80, 200, 60, 220],
- mediaType: 'img',
- mediaCallback: (rowId) => url,
- },
-});
-```
-
-#### view style
-
-Specify the visual style of the view.
-
-- width - canvas width
-- height - canvas height
-- padding - the main view padding
-
-#### media supports
-
-Support mapping from Row No. to media files. (current support img)
-
-- mediaType - null | img
-- mediaCallback - func: rowId => url,
-
 ## Examples
 
-We prepare a simple case, which is an overview of an hnsw with 17,000+ vectors.
+We prepare a simple case, which is the visualizations of the `hnsw` and `ivf_flat` with 17,000+ vectors that embedded from [VOC 2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar)).
+
 Only need enable a web service.
 
 ```shell
@@ -121,41 +98,13 @@ python -m http.server
 
 Then open http://localhost:8000/
 
-- If you want to explore the search process of hnsw, you can modify **test/test.js**.
-
- **searchRandTestVec** will randomly select a vector from the index file as the target vector.
-
- ```js
- window.addEventListener('DOMContentLoaded', async () => {
- ...
- // feder.overview();
- feder.searchRandTestVec();
- ...
- });
- ```
-
- then open a new cmdline,
+It will show 4 visualizations:
+- `hnsw` overview
+- `hnsw` search view
+- `ivf_flat` overview
+- `ivf_flat` search view
 
- ```shell
- yarn dev
- ```
-
- It makes the new changes to test.js take effect.
-
-- If you want to display the image during the interaction, you can modify **test/test.js**, and use the **testHNSWWithImages** function.
-
- ```js
- window.addEventListener('DOMContentLoaded', async () => {
- ...
- // const feder = await testHNSW('https://assets.zilliz.com/hnswlib_hnsw_voc_17k_1f1dfd63a9.index');
- const feder = await testHNSWWithImages('https://assets.zilliz.com/hnswlib_hnsw_voc_17k_1f1dfd63a9.index');
- ...
- });
- ```
-
-- If you want to use a new dataset, the following process will help you.
-
-## Explore a new data
+## Pipeline - explore a new dataset with feder
 
 ### Step 1. Dataset preparation
 
@@ -175,7 +124,7 @@ You can use [faiss](https://github.com/facebookresearch/faiss) or [hnswlib](http
 
 (\*Detailed procedures please refer to their tutorials.)
 
-Referring to **test/data/gen*hnswlib_index*\*.py** or **test/data/gen*faiss_index*\*.py**
+Referring to **test/data/gen_hnswlib_index_\*.py** or **test/data/gen_faiss_index_\*.py**
 
 Or we have the [index file](https://assets.zilliz.com/hnswlib_hnsw_voc_17k_1f1dfd63a9.index) ready for you.
 
@@ -244,9 +193,8 @@ More cases refer to the **test/test.js**
 - [Visualize Your Approximate Nearest Neighbor Search with Feder](https://zilliz.com/blog/Visualize-Your-Approximate-Nearest-Neighbor-Search-with-Feder)
 - [Visualize Reverse Image Search with Feder](https://zilliz.com/blog/Visualize-Reverse-Image-Search-with-Feder)
 
-
-
 ## Roadmap
+
 We're still in the early stages, we will support more types of anns index, and more unstructured data viewer, stay tuned.
 
 ## Acknowledgments

federjs/FederView/IvfflatView/index.js

@@ -110,9 +110,11 @@ export default class IvfflatView extends BaseView {
  infoPanel.updateOverviewHoveredInfo({
  hoveredCluster,
  listIds: this.indexMeta.listIds[hoveredClusterId],
- images: this.indexMeta.listIds[hoveredClusterId].map((listId) =>
- this.mediaCallback(listId)
- ),
+ images: this.mediaCallback
+ ? this.indexMeta.listIds[hoveredClusterId].map((listId) =>
+ this.mediaCallback(listId)
+ )
+ : [],
  x: hoveredCluster.OVPolyCentroid[0] / this.canvasScale,
  y: hoveredCluster.OVPolyCentroid[1] / this.canvasScale,
  });
@@ -310,9 +312,11 @@ export default class IvfflatView extends BaseView {
  infoPanel.updateSearchViewHoveredInfo({
  hoveredCluster,
  listIds: this.indexMeta.listIds[hoveredClusterId],
- images: this.indexMeta.listIds[hoveredClusterId].map((listId) =>
- this.mediaCallback(listId)
- ),
+ images: this.mediaCallback
+ ? this.indexMeta.listIds[hoveredClusterId].map((listId) =>
+ this.mediaCallback(listId)
+ )
+ : [],
  x: hoveredCluster.SVPolyCentroid[0] / this.canvasScale,
  y: hoveredCluster.SVPolyCentroid[1] / this.canvasScale,
  });
@@ -343,7 +347,10 @@ export default class IvfflatView extends BaseView {
  );
  }
 
- const img = hoveredNode ? this.mediaCallback(hoveredNode.id) : '';
+ const img =
+ hoveredNode && this.mediaCallback
+ ? this.mediaCallback(hoveredNode.id)
+ : '';
  infoPanel.updateSearchViewHoveredNodeInfo({
  hoveredNode,
  img,

federpy/README.md

@@ -4,10 +4,20 @@
 
 Support search-by-vector
 
-*searchByVec(vector [, imgUrl=None, isDisplayed=True])*
+```
+searchByVec(vector [, imgUrl=None, isDisplayed=True])
+```
 
 ## 0.7.0
 
 Each action (search or overview) command generates a new div with a random id.
 
-Different actions will no longer share an output cell.
+Different actions will no longer share an output cell.
+
+## 0.8.0
+
+Support chain functions.
+
+```
+federPy.setSearchParams(params).search()
+```

federpy/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
  name="federpy",
- version="0.7.0",
+ version="0.8.0",
  author="min.tian@zilliz",
  author_email="[email protected]",
  description="Feder for python",

federpy/src/federpy/federpy.py

@@ -71,6 +71,7 @@ def searchRandTestVec(self, isDisplay=True):
 
  def setSearchParams(self, searchParams):
  self.searchParams = searchParams
+ return self
 
  def getJs(self):
  return """

package.json

@@ -1,7 +1,7 @@
 {
  "name": "@zilliz/feder",
  "author": "[email protected]",
- "version": "0.2.2",
+ "version": "0.2.3",
  "description": "visualization packages for vector space",
  "main": "dist/index.js",
  "files": [

test/bundle.js

@@ -17633,7 +17633,7 @@ ${indentData}`);
  infoPanel.updateOverviewHoveredInfo({
  hoveredCluster,
  listIds: this.indexMeta.listIds[hoveredClusterId],
- images: this.indexMeta.listIds[hoveredClusterId].map((listId) => this.mediaCallback(listId)),
+ images: this.mediaCallback ? this.indexMeta.listIds[hoveredClusterId].map((listId) => this.mediaCallback(listId)) : [],
  x: hoveredCluster.OVPolyCentroid[0] / this.canvasScale,
  y: hoveredCluster.OVPolyCentroid[1] / this.canvasScale
  });
@@ -17737,7 +17737,7 @@ ${indentData}`);
  infoPanel.updateSearchViewHoveredInfo({
  hoveredCluster,
  listIds: this.indexMeta.listIds[hoveredClusterId],
- images: this.indexMeta.listIds[hoveredClusterId].map((listId) => this.mediaCallback(listId)),
+ images: this.mediaCallback ? this.indexMeta.listIds[hoveredClusterId].map((listId) => this.mediaCallback(listId)) : [],
  x: hoveredCluster.SVPolyCentroid[0] / this.canvasScale,
  y: hoveredCluster.SVPolyCentroid[1] / this.canvasScale
  });
@@ -17758,7 +17758,7 @@ ${indentData}`);
  hoveredNode = currentHoveredNode;
  this.renderFineSearch(ctx, infoPanel, searchViewLayoutData, searchViewType, hoveredNode);
  }
- const img = hoveredNode ? this.mediaCallback(hoveredNode.id) : "";
+ const img = hoveredNode && this.mediaCallback ? this.mediaCallback(hoveredNode.id) : "";
  infoPanel.updateSearchViewHoveredNodeInfo({
  hoveredNode,
  img,
@@ -17951,42 +17951,60 @@ ${indentData}`);
  }
  };
 
- // test/test.js
- var domSelector = "#container";
- var getId2name = () => __async(void 0, null, function* () {
- const data = yield csv2("https://assets.zilliz.com/voc_names_4cee9440b1.csv");
- const rowId2name = {};
- data.forEach((d, i) => rowId2name[i] = d.name);
+ // test/config.js
+ var hnswSource = "hnswlib";
+ var hnswIndexFilePath = "https://assets.zilliz.com/hnswlib_hnsw_voc_17k_1f1dfd63a9.index";
+ var ivfflatSource = "faiss";
+ var ivfflatIndexFilePath = "https://assets.zilliz.com/faiss_ivf_flat_voc_17k_ab112eec72.index";
+ var imgNamesFilePath = "https://assets.zilliz.com/voc_names_4cee9440b1.csv";
+ var getRowId2name = () => __async(void 0, null, function* () {
+ const data = yield csv2(imgNamesFilePath);
+ const rowId2name = (rowId) => data[rowId].name;
  return rowId2name;
  });
- var testIVFFlatWithImages = (filePath) => __async(void 0, null, function* () {
- const rowId2name = yield getId2name();
- const mediaCallback = (rowId) => rowId in rowId2name ? `https://assets.zilliz.com/voc2012/JPEGImages/${rowId2name[rowId]}` : null;
+ var name2imgUrl = (name) => `https://assets.zilliz.com/voc2012/JPEGImages/${name}`;
+ var getRowId2imgUrl = () => __async(void 0, null, function* () {
+ const rowId2name = yield getRowId2name();
+ const rowId2imgUrl = (rowId) => name2imgUrl(rowId2name(rowId));
+ return rowId2imgUrl;
+ });
+
+ // test/testHnsw.js
+ var getFederHnsw = () => __async(void 0, null, function* () {
+ const rowId2imgUrl = yield getRowId2imgUrl();
  const feder = new Feder({
- filePath,
- source: "faiss",
+ source: hnswSource,
+ filePath: hnswIndexFilePath,
  viewParams: {
- height: 300,
  mediaType: "img",
- mediaCallback,
- projectSeed: 1235,
- projectMethod: "umap"
+ mediaCallback: rowId2imgUrl
  }
  });
  return feder;
  });
- window.addEventListener("DOMContentLoaded", () => __async(void 0, null, function* () {
- const feder = yield testIVFFlatWithImages("https://assets.zilliz.com/faiss_ivf_flat_voc_17k_ab112eec72.index");
- console.log(feder);
- feder.setSearchParams({
- k: 12,
- nprobe: 8,
- ef: 10
+
+ // test/testIvfflat.js
+ var getFederIvfflat = () => __async(void 0, null, function* () {
+ const rowId2imgUrl = yield getRowId2imgUrl();
+ const feder = new Feder({
+ source: ivfflatSource,
+ filePath: ivfflatIndexFilePath,
+ viewParams: {
+ mediaType: "img",
+ mediaCallback: rowId2imgUrl
+ }
  });
- document.querySelector(domSelector).appendChild(feder.overview());
- document.querySelector(domSelector).appendChild(feder.searchRandTestVec());
- document.querySelector(domSelector).appendChild(feder.setSearchParams({ k: 4, nprobe: 6, ef: 6 }).searchById(4365));
- document.querySelector(domSelector).appendChild(feder.setSearchParams({ k: 6, nprobe: 10, ef: 8 }).searchRandTestVec());
- document.querySelector(domSelector).appendChild(feder.overview());
+ return feder;
+ });
+
+ // test/test.js
+ var domSelector = "#container";
+ window.addEventListener("DOMContentLoaded", () => __async(void 0, null, function* () {
+ const hnsw_feder = yield getFederHnsw();
+ document.querySelector(domSelector).appendChild(hnsw_feder.overview());
+ document.querySelector(domSelector).appendChild(hnsw_feder.setSearchParams({ k: 6, nprobe: 8, ef: 9 }).searchRandTestVec());
+ const ivfflat_feder = yield getFederIvfflat();
+ document.querySelector(domSelector).appendChild(ivfflat_feder.overview());
+ document.querySelector(domSelector).appendChild(ivfflat_feder.setSearchParams({ k: 4, nprobe: 6, ef: 6 }).searchById(4365));
  }));
 })();