refactoring rest-server && model-hub && model-client

open-intelligence · Dec 30, 2018 · 6c7f440 · 6c7f440
1 parent ad4dbb3
commit 6c7f440
Show file tree

Hide file tree

Showing 183 changed files with 20,717 additions and 10,275 deletions.
diff --git a/README_zh.md b/README_zh.md
@@ -78,9 +78,10 @@ Web门户上也提供了Web UI进行集群的管理。
 ## 系统结构
 
 <p style="text-align: left;">
- <img src="./sysarch.png" title="System Architecture" alt="System Architecture" />
+ <img src="./sysarch-zh.png" title="System Architecture" alt="System Architecture" />
 </p>
 
+
 系统的整体结构如上图所示。
 用户通过[Web门户](./webportal/README.md)提交了任务作业或集群状态监视的申请，该操作会调用[REST服务器](./rest-server/README.md)提供的API。
 第三方工具也可以直接调用REST服务器进行作业管理。收到API调用后，REST服务器与[FrameworkLauncher](./frameworklauncher/README.md)（简称Launcher）协同工作来进行作业管理。Launcher服务器处理来自REST服务器的请求，并将任务作业提交到Hadoop YARN。由YARN和[GPU强化](https://issues.apache.org/jira/browse/YARN-7481)调度的作业, 可以使用集群中的GPU资源进行深度学习运算。其他基于CPU的AI工作或者传统的大数据任务作业也可以在平台上运行，与那些基于GPU的作业共存。

diff --git a/frameworklauncher/README-zh.md b/frameworklauncher/README-zh.md
@@ -16,7 +16,7 @@
  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
- 
+
  Copyright (c) Peking University 2018
 
  The software is released under the Open-Intelligence Open Source License V1.0.
@@ -25,7 +25,7 @@
  Artificial Intelligence Technology Innovation Strategic Alliance (the AITISA).
 -->
 
-# 微软FrameworkLauncher
+# FrameworkLauncher
 
 搭建FrameworkLauncher（简称Launcher）是为了支持在 [YARN](http://hadoop.apache.org/) 容器内可以长期运行大规模服务，而无需更改该服务自身。 它还支持批量作业，如TensorFlow，CNTK等。
 

diff --git a/frameworklauncher/README.md b/frameworklauncher/README.md
@@ -17,7 +17,7 @@
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 -->
 
-# Microsoft FrameworkLauncher
+# FrameworkLauncher
 
 FrameworkLauncher (or Launcher for short) is built to enable running Large-Scale Long-Running Services inside [YARN](http://hadoop.apache.org/) Containers without making changes to the Services themselves. It also supports Batch Jobs, such as TensorFlow, CNTK, etc.
 

diff --git a/model-client/command/commands/clone.js b/model-client/command/commands/clone.js
@@ -10,7 +10,7 @@ const fileUtil = require("../../utils/file");
 const lang = require("../../prototype/lang");
 const log = require("../../utils/log");
 const validate = require("../../utils/validate");
-const userInfo = require("../../lib/userInfo");
+const userLib = require("../../lib/user");
 
 function isExist(dir,project_name){
  let project_path = path.join(dir,project_name);
@@ -34,7 +34,6 @@ function GetVersions(user,project,server){
  if(err){
  throw err;
  }
-
 
  if( body instanceof Buffer){
  body = body.toString();
@@ -48,6 +47,7 @@ function GetVersions(user,project,server){
  access_denied:false,
  versions:{}
  };
+
  if( body.success == true){
  response.success = true;
  response.versions = body.versions;
@@ -60,7 +60,6 @@ function GetVersions(user,project,server){
  response.internal_error = true;
  }
  }
-
 
  return response;
  });
@@ -75,13 +74,11 @@ function clone(args,opt){
  let cwd = process.cwd();
 
  let project_address = args[0];
-
- console.log(project_address)
 
  if(!validate.isValidateProjectAddress(project_address)){
  log.info(lang.New()
- .zh("[CLONE ERROR] - 项目地址格式不正确")
- .en("[CLONE ERROR] - Wrong form of repository address!"));
+ .zh("项目地址格式不正确")
+ .en("Wrong form of repository address!"));
 
  log.info(lang.New()
  .zh("正确的格式是'http(s)://host/username/project_name.ms'")
@@ -120,7 +117,7 @@ function clone(args,opt){
  return false;
  }
 
- let [err1,user_info] = yield userInfo.GetInfo(server.hostname,user);
+ let [err1,user_info] = yield userLib.getInfo(server.hostname,user);
 
  if(err1){
  throw err1;
@@ -139,19 +136,19 @@ function clone(args,opt){
  if(!response.success){
 
  if(response.internal_error == true){
- log.info("[CLONE ERROR] - "+response.message);
+ log.info(response.message);
+ }else{
+ log.info(lang.New().en(`Access denied,Please login again.`).zh("鉴权失败，请登录远程仓库"));
  }
 
- log.info(lang.New().en(`[CLONE ERROR] - Access denied,Please login again.`).zh("[CLONE ERROR] - 鉴权失败，请登录远程仓库"));
-
  return false;
 
  }
 
  if(response.versions.length == 0){
  log.info(lang.New()
- .en(`[CLONE ERROR] - Project "${project_name}" is not found at remote!`)
- .zh(`[CLONE ERROR] - 项目"${project_name}"不存在`))
+ .en(`Project "${project_name}" is not found at remote!`)
+ .zh(`项目"${project_name}"不存在`))
 
  return false;
  }
@@ -170,8 +167,8 @@ function clone(args,opt){
  }
  if(!found){
  log.info(lang.New()
- .en(`[CLONE ERROR] - The version "${version}" of "${project_name}" is not found`)
- .zh(`[CLONE ERROR] - 未找到${version}版的${project_name}`));
+ .en(`The version "${version}" of "${project_name}" is not found`)
+ .zh(`未找到${version}版的${project_name}`));
 
 
  return false;

diff --git a/model-client/command/commands/init.js b/model-client/command/commands/init.js
@@ -45,8 +45,8 @@ function init(args,opt){
  break;
  }else{
  log.info(lang.New()
- .zh("[INIT ERROR] - 项目地址格式不正确")
- .en("[INIT ERROR] - Wrong form of repository address!"));
+ .zh("项目地址格式不正确")
+ .en("Wrong form of repository address!"));
 
  log.info(lang.New()
  .zh("正确的格式是'http(s)://host/username/project_name.ms'")

diff --git a/model-client/command/commands/login.js b/model-client/command/commands/login.js
@@ -2,8 +2,7 @@
 const co = require("zco");
 const Command = require("../../prototype/command");
 const lang = require("../../prototype/lang");
-const login = require("../../lib/login");
-const userInfo = require("../../lib/userInfo");
+const userLib = require("../../lib/user")
 const url = require("url");
 const log = require('../../utils/log');
 
@@ -43,10 +42,10 @@ function Login (args,opt){
 
  this.ctx.server = server;
 
- let loginRes = yield login.login(user,pwd);
+ let loginRes = yield userLib.login(user,pwd);
 
  if(loginRes.success){
- yield userInfo.SetInfo(server.hostname,loginRes.token,user);
+ yield userLib.setInfo(server.hostname,loginRes.token,user);
  log.info(lang.New().en(" Login successfully!").zh("登录成功"));
 
  }else{

diff --git a/model-client/command/commands/touch.js b/model-client/command/commands/touch.js
@@ -8,8 +8,6 @@ const Command = require("../../prototype/command");
 
 const lang = require("../../prototype/lang");
 
-
-
 function touch (args){
  return co.brief(function*(resume){
 

diff --git a/model-client/command/commands/version.js b/model-client/command/commands/version.js
@@ -14,7 +14,7 @@ function create(cwd,args,opt){
  let version = args[1];
 
  if (!version){
- log.info("[VERSION CREATE ERROR] - "+lang.New().en("Parameter 'version' is required!").zh("缺失参数'version'(版本号)")) 
+ log.info(lang.New().en("Parameter 'version' is required!").zh("缺失参数'version'(版本号)")) 
  return null;
  }
 
@@ -33,7 +33,7 @@ function create(cwd,args,opt){
  }
 
  if(json.versions.includes(version)){
- log.info("[VERSION CREATE ERROR] "+lang.New().en(`Version "${version}" has already existed.`).zh(`版本"${version}"已经存在`));
+ log.info(lang.New().en(`Version "${version}" has already existed.`).zh(`版本"${version}"已经存在`));
  return null;
  }
 
@@ -83,7 +83,7 @@ function create(cwd,args,opt){
  yield fileUtil.copy(src_path,cwd,[path.join(src_path,"project.json")]);
  }
 
- log.info("[VERSION CREATE] - "+lang.New().en(`Create new version "${version}" successfully!`).zh(`版本${version}创建成功`));
+ log.info(lang.New().en(`Create new version "${version}" successfully!`).zh(`版本${version}创建成功`));
 
  });
 }
@@ -92,8 +92,7 @@ function _switch(cwd,version){
  return co.brief(function*(resume){
  let [err,file] = yield fs.readFile(path.join(cwd,"project.json"),resume);
  if(err){
- log.info("[]VERSION SWITCH ERROR] - "+err.message);
-
+ log.info(lang.New().en("Failed to switch version,"+err.message).zh("切换版本失败,"+err.message));
  return false;
  }
  let json = JSON.parse(file);
@@ -102,20 +101,20 @@ function _switch(cwd,version){
  return null;
  }
  if(!json.versions.includes(version)){
- log.info("[VERSION SWITCH ERROR] - "+lang.New().en(`The version "${version}" has not existed!`).zh(`版本"${version}"不存在`));
+ log.info(lang.New().en(`The version "${version}" has not existed!`).zh(`版本"${version}"不存在`));
 
  return null;
  }
  let target_version_cache_path = path.join(cwd,".ms","v"+version);
 
  if(!fs.existsSync(target_version_cache_path)){
- log.info("[VERSION SWITCH ERROR] - "+lang.New().en(`The version "${version}" has not existed!`).zh(`版本"${version}"不存在`));
+ log.info(lang.New().en(`The version "${version}" has not existed!`).zh(`版本"${version}"不存在`));
 
  return null;
  }
 
  if(!fs.existsSync(path.join(target_version_cache_path,"project.json"))){
- log.info("[VERSION SWITCH ERROR] - "+lang.New().en("Missing 'project.json'.").zh("缺失'project.json'."));
+ log.info(lang.New().en("Missing 'project.json'.").zh("缺失'project.json'."));
  return null;
  }
 
@@ -128,7 +127,7 @@ function _switch(cwd,version){
 
  if(err){
 
- log.info("[VERSION SWITCH ERROR] - "+err.message);
+ log.info(lang.New().en("Failed to switch version,"+err.message).zh("切换版本失败,"+err.message));
  return false;
 
  }
@@ -142,14 +141,14 @@ function _switch(cwd,version){
  [err,file] = yield fs.readFile(path.join(cwd,"project.json"),resume);
 
  if(err){
- log.info("[VERSION SWITCH ERROR] - "+err.message);
+ log.info(lang.New().en("Failed to switch version,"+err.message).zh("切换版本失败,"+err.message));
  return false;
  }
 
  json = JSON.parse(file);
 
  if(false == json.synced){
- log.info("[VERSION SWITCH] - "+lang.New().en("Start to download data from remote repository server\n").zh("开始从远程仓库下载项目数据"));
+ log.info(lang.New().en("Start to download data from remote repository server\n").zh("开始从远程仓库下载项目数据"));
  yield download.download();
  delete json.synced;
  [err] = yield fs.writeFile(path.join(cwd,"project.json"),JSON.stringify(json," ",2),resume);
@@ -158,7 +157,7 @@ function _switch(cwd,version){
  }
  }
 
- log.info(lang.New().en("[VERSION SWITCH] - Successfully").zh("成功切换版本"));
+ log.info(lang.New().en("Successfully").zh("成功切换版本"));
 
  });
 }
@@ -167,7 +166,7 @@ function current(cwd){
  return co.brief(function*(resume){
  let [err,file] = yield fs.readFile(path.join(cwd,"project.json"),resume);
  if(err){
- console.log("[VERSION LIST ERROR] - "+err.message);
+ log.info(lang.New().en("Failed to print current version,"+err.message).zh("打印当前版本号失败,"+err.message))
  return false;
  }
  let json = JSON.parse(file);
@@ -180,7 +179,8 @@ function list(cwd){
  return co.brief(function*(resume){
  let [err,file] = yield fs.readFile(path.join(cwd,"project.json"),resume);
  if(err){
- console.log("[VERSION LIST ERROR] - "+err.message);
+ log.info(lang.New().en("Failed to list versions,"+err.message).zh("列出版本列表失败,"+err.message))
+
  return false;
  }
  let json = JSON.parse(file);
@@ -209,21 +209,25 @@ function version(args,opt){
  yield current(cwd);
  return false;
  }
+
  let sub_command = args[0];
 
  if("create" == sub_command){
+
  yield create(cwd,args,opt);
+
  }else if("switch" == sub_command){
- if (!opt.v){
- log.info("[VERSION SWITCH ERROR] - "+lang.New().en("Parameter 'version' is required!").zh("缺少'版本号'参数"));
 
-
+ if (!opt.v){
+ log.info(lang.New().en("Parameter 'version' is required!").zh("缺少'版本号'参数"));
  }else{
  yield _switch(cwd,opt.v);
  }
 
  }else if("list" == sub_command){
+
  yield list(cwd,opt);
+
  }else{
  log.info(lang.New().en(`Command "version ${sub_command}" is not found`).zh(`无此命令 "version ${sub_command}"`))
 
@@ -234,6 +238,7 @@ function version(args,opt){
 }
 
 let type = lang.New().en("About Project Management").zh("项目管理相关");
+
 let note = lang.New().en("List,create,or switch versions.").zh("罗列,创建,或者切换版本");
 
 const cmd = new Command("version",version,type);
@@ -248,7 +253,7 @@ cmd.usage("version switch -v 'version'",lang.New().en("Switch to target version.
 
 cmd.usage("version list",lang.New().en("List all versions.").zh("列出所有的版本"));
 
-cmd.usage("version",lang.New().en("Print the current versio.n").zh("打印出当前版本号"));
+cmd.usage("version",lang.New().en("Print the current version.").zh("打印出当前版本号"));
 
 module.exports = cmd;
 
diff --git a/model-client/command/index.js b/model-client/command/index.js
@@ -23,9 +23,9 @@ const cmd_exit = require("./commands/exit");
 const cmd_lang = require("./commands/lang");
 
 
-const exist = require("./middlewares/exist")
-const notFound = require("./middlewares/not_found");
-const signed = require("./middlewares/signed");
+const exist = require("./middlewares/project_exist")
+const not_found = require("./middlewares/not_found");
+const check_login = require("./middlewares/check_login");
 
 require("./setup").run();
 
@@ -44,15 +44,13 @@ flow.use("touch",cmd_touch)//done
 flow.use("init",cmd_init);//done
 flow.use("login",cmd_login);//done
 flow.use("clone",cmd_clone);//done
-flow.use("pull",exist,signed,cmd_pull);//done
-flow.use("push",exist,signed,cmd_push);//done
+flow.use("pull",exist,check_login,cmd_pull);//done
+flow.use("push",exist,check_login,cmd_push);//done
 flow.use("version",exist,cmd_version);//done
-flow.use("convert",exist,signed,cmd_convert);//done
+flow.use("convert",exist,check_login,cmd_convert);//done
 
 
-
-
-flow.use(notFound);
+flow.use(not_found);
 
 
 co(function*(resume){