app.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from flask import Flask, request, render_template, send_from_directory, jsonify, session, send_file
  15. from werkzeug.utils import secure_filename
  16. from flask_cors import CORS
  17. import argparse
  18. from os import path as osp
  19. import os
  20. import time
  21. import json
  22. import sys
  23. import multiprocessing as mp
  24. from . import workspace_pb2 as w
  25. from .utils import CustomEncoder, ShareData, is_pic, get_logger, TaskStatus
  26. import numpy as np
  27. app = Flask(__name__)
  28. CORS(app, supports_credentials=True)
  29. SESSION_TYPE = 'filesystem'
  30. app.config.from_object(__name__)
  31. SD = ShareData()
  32. def init(dirname, logger):
  33. #初始化工作空间
  34. from .workspace import init_workspace
  35. from .system import get_system_info
  36. SD.workspace = w.Workspace(path=dirname)
  37. init_workspace(SD.workspace, dirname, logger)
  38. SD.workspace_dir = dirname
  39. get_system_info(SD.machine_info)
  40. @app.errorhandler(Exception)
  41. def handle_exception(e):
  42. ret = {"status": -1, 'message': repr(e)}
  43. return ret
  44. @app.route('/workspace', methods=['GET', 'PUT'])
  45. def workspace():
  46. """
  47. methods=='GET':获取工作目录中项目、数据集、任务的属性
  48. Args:
  49. struct(str):结构类型,可以是'dataset', 'project'或'task',
  50. id(str):结构类型对应的id
  51. attr_list(list):需要获取的属性的列表
  52. Return:
  53. attr(dict):key为属性,value为属性的值,
  54. status
  55. methods=='PUT':修改工作目录中项目、数据集、任务的属性
  56. Args:
  57. struct(str):结构类型,可以是'dataset', 'project'或'task',
  58. id(str):结构类型对应的id
  59. attr_dict(dict):key:需要修改的属性,value:需要修改属性的值
  60. Return:
  61. status
  62. """
  63. data = request.get_json()
  64. if data is None:
  65. data = request.args
  66. if request.method == 'GET':
  67. if data:
  68. from .workspace import get_attr
  69. ret = get_attr(data, SD.workspace)
  70. return ret
  71. return {'status': 1, 'dirname': SD.workspace_dir}
  72. if request.method == 'PUT':
  73. from .workspace import set_attr
  74. ret = set_attr(data, SD.workspace)
  75. return ret
  76. @app.route('/dataset', methods=['GET', 'POST', 'PUT', 'DELETE'])
  77. def dataset():
  78. """
  79. methods=='GET':获取所有数据集或者单个数据集的信息
  80. Args:
  81. did(str, optional):数据集id(可选),如果存在就返回数据集id对应数据集的信息
  82. Ruturn:
  83. status
  84. if 'did' in Args:
  85. id(str):数据集id,
  86. dataset_status(int):数据集状态(DatasetStatus)枚举变量的值
  87. message(str):数据集状态信息
  88. attr(dict):数据集属性
  89. else:
  90. datasets(list):所有数据集属性的列表
  91. methods=='POST':创建一个新的数据集
  92. Args:
  93. name(str):数据集名字
  94. desc(str):数据集描述
  95. dataset_type(str):数据集类型,可以是['classification', 'detection', 'segmentation','instance_segmentation','remote_segmentation']
  96. Return:
  97. did(str):数据集id
  98. status
  99. methods=='PUT':异步,向数据集导入数据,支持分类、检测、语义分割、实例分割、摇杆分割数据集类型
  100. Args:
  101. did(str):数据集id
  102. path(str):数据集路径
  103. Return:
  104. status
  105. methods=='DELETE':删除已有的某个数据集
  106. Args:
  107. did(str):数据集id
  108. Return:
  109. status
  110. """
  111. data = request.get_json()
  112. if data is None:
  113. data = request.args
  114. if request.method == 'GET':
  115. if 'did' in data:
  116. from .dataset.dataset import get_dataset_status
  117. ret = get_dataset_status(data, SD.workspace)
  118. return ret
  119. from .dataset.dataset import list_datasets
  120. ret = list_datasets(SD.workspace)
  121. return ret
  122. if request.method == 'POST':
  123. from .dataset.dataset import create_dataset
  124. ret = create_dataset(data, SD.workspace)
  125. return ret
  126. if request.method == 'PUT':
  127. from .dataset.dataset import import_dataset
  128. ret = import_dataset(data, SD.workspace, SD.monitored_processes,
  129. SD.load_demo_proc_dict)
  130. return ret
  131. if request.method == 'DELETE':
  132. from .dataset.dataset import delete_dataset
  133. ret = delete_dataset(data, SD.workspace)
  134. return ret
  135. @app.route('/dataset/details', methods=['GET'])
  136. def dataset_details():
  137. """
  138. methods=='GET':获取某个数据集的详细信息
  139. Args:
  140. did(str):数据集id
  141. Return:
  142. details(dict):数据集详细信息,
  143. status
  144. """
  145. data = request.get_json()
  146. if data is None:
  147. data = request.args
  148. if request.method == 'GET':
  149. from .dataset.dataset import get_dataset_details
  150. ret = get_dataset_details(data, SD.workspace)
  151. return ret
  152. @app.route('/dataset/split', methods=['PUT'])
  153. def dataset_split():
  154. """
  155. Args:
  156. did(str):数据集id
  157. val_split(float): 验证集比例
  158. test_split(float): 测试集比例
  159. Return:
  160. status
  161. """
  162. data = request.get_json()
  163. if request.method == 'PUT':
  164. from .dataset.dataset import split_dataset
  165. ret = split_dataset(data, SD.workspace)
  166. return ret
  167. @app.route('/dataset/image', methods=['GET'])
  168. def dataset_img_base64():
  169. """
  170. Args:
  171. GET: 获取图片base64数据,参数:'path' 图片绝对路径
  172. """
  173. data = request.get_json()
  174. if request.method == 'GET':
  175. from .dataset.dataset import img_base64
  176. ret = img_base64(data)
  177. return ret
  178. @app.route('/dataset/file', methods=['GET'])
  179. def get_image_file():
  180. """
  181. Args:
  182. GET: 获取文件数据,参数:'path' 文件绝对路径
  183. """
  184. data = request.get_json()
  185. if request.method == 'GET':
  186. ret = data['path']
  187. return send_file(ret)
  188. @app.route('/dataset/npy', methods=['GET'])
  189. def get_npyfile():
  190. """
  191. Args:
  192. GET: 获取文件数据,参数:'path' npy文件绝对路径
  193. """
  194. data = request.get_json()
  195. if request.method == 'GET':
  196. npy = np.load(data['path'], allow_pickle=True).tolist()
  197. npy['gt_bbox'] = npy['gt_bbox'].tolist()
  198. return npy
  199. @app.route('/file', methods=['GET'])
  200. def get_file():
  201. """
  202. Args:
  203. path'(str):文件在服务端的路径
  204. Return:
  205. #数据为图片
  206. img_data(str): base64图片数据
  207. status
  208. #数据为xml文件
  209. ret:数据流
  210. #数据为log文件
  211. ret:json数据
  212. """
  213. data = request.get_json()
  214. if data is None:
  215. data = request.args
  216. if request.method == 'GET':
  217. path = data['path']
  218. if not os.path.exists(path):
  219. return {'status': -1}
  220. if is_pic(path):
  221. from .dataset.dataset import img_base64
  222. ret = img_base64(data, SD.workspace)
  223. return ret
  224. file_type = path[(path.rfind('.') + 1):]
  225. if file_type in ['xml', 'npy', 'log']:
  226. return send_file(path)
  227. else:
  228. pass
  229. @app.route('/project', methods=['GET', 'POST', 'DELETE'])
  230. def project():
  231. """
  232. methods=='GET':获取指定项目id的信息
  233. Args:
  234. 'id'(str, optional):项目id,可选,如果存在就返回项目id对应项目的信息
  235. Return:
  236. status,
  237. if 'id' in Args:
  238. attr(dict):项目属性
  239. else:
  240. projects(list):所有项目属性
  241. methods=='POST':创建一个项目
  242. Args:
  243. name(str): 项目名
  244. desc(str):项目描述
  245. project_type(str):项目类型
  246. Return:
  247. pid(str):项目id
  248. status
  249. methods=='DELETE':删除一个项目,以及项目相关的task
  250. Args:
  251. pid(str):项目id
  252. Return:
  253. status
  254. """
  255. data = request.get_json()
  256. if data is None:
  257. data = request.args
  258. if request.method == 'GET':
  259. from .project.project import list_projects
  260. from .project.project import get_project
  261. if 'id' in data:
  262. ret = get_project(data, SD.workspace)
  263. return ret
  264. ret = list_projects(SD.workspace)
  265. return ret
  266. if request.method == 'POST':
  267. from .project.project import create_project
  268. ret = create_project(data, SD.workspace)
  269. return ret
  270. if request.method == 'DELETE':
  271. from .project.project import delete_project
  272. ret = delete_project(data, SD.workspace)
  273. return ret
  274. @app.route('/project/task', methods=['GET', 'POST', 'DELETE'])
  275. def task():
  276. """
  277. methods=='GET':#获取某个任务的信息或者所有任务的信息
  278. Args:
  279. tid(str, optional):任务id,可选,若存在即返回id对应任务的信息
  280. resume(str, optional):获取是否可以恢复训练的状态,可选,需在存在tid的情况下才生效
  281. pid(str, optional):项目id,可选,若存在即返回该项目id下所有任务信息
  282. Return:
  283. status
  284. if 'tid' in Args:
  285. task_status(int):任务状态(TaskStatus)枚举变量的值
  286. message(str):任务状态信息
  287. type:任务类型包括{'classification', 'detection', 'segmentation', 'instance_segmentation'}
  288. resumable(bool):仅Args中存在resume时返回,任务训练是否可以恢复
  289. max_saved_epochs(int):仅Args中存在resume时返回,当前训练模型保存的最大epoch
  290. else:
  291. tasks(list):所有任务属性
  292. methods=='POST':#创建任务(训练或者裁剪)
  293. Args:
  294. pid(str):项目id
  295. train(dict):训练参数
  296. desc(str, optional):任务描述,可选
  297. parent_id(str, optional):可选,若存在即表示新建的任务为裁剪任务,parent_id的值为裁剪任务对应的训练任务id
  298. Return:
  299. tid(str):任务id
  300. status
  301. methods=='DELETE':#删除任务
  302. Args:
  303. tid(str):任务id
  304. Return:
  305. status
  306. """
  307. data = request.get_json()
  308. if data is None:
  309. data = request.args
  310. if request.method == 'GET':
  311. if data:
  312. if 'pid' not in data:
  313. from .project.task import get_task_status
  314. ret = get_task_status(data, SD.workspace)
  315. return ret
  316. from .project.task import list_tasks
  317. ret = list_tasks(data, SD.workspace)
  318. return ret
  319. if request.method == 'POST':
  320. from .project.task import create_task
  321. ret = create_task(data, SD.workspace)
  322. return ret
  323. if request.method == 'DELETE':
  324. from .project.task import delete_task
  325. ret = delete_task(data, SD.workspace)
  326. return ret
  327. @app.route('/project/task/params', methods=['GET', 'POST'])
  328. def task_params():
  329. """
  330. methods=='GET':#获取任务id对应的参数,或者获取项目默认参数
  331. Args:
  332. tid(str, optional):获取任务对应的参数
  333. pid(str,optional):获取项目对应的默认参数
  334. model_type(str,optional):pid存在下有效,对应项目下获取指定模型的默认参数
  335. gpu_list(list,optional):pid存在下有效,默认值为[0],使用指定的gpu并获取相应的默认参数
  336. Return:
  337. train(dict):训练或者裁剪的参数
  338. status
  339. methods=='POST':#设置任务参数,将前端用户设置训练参数dict保存在后端的pkl文件中
  340. Args:
  341. tid(str):任务id
  342. train(dict):训练参数
  343. Return:
  344. status
  345. """
  346. data = request.get_json()
  347. if data is None:
  348. data = request.args
  349. if request.method == 'GET':
  350. if 'tid' in data:
  351. from .project.task import get_task_params
  352. ret = get_task_params(data, SD.workspace)
  353. ret['train'] = CustomEncoder().encode(ret['train'])
  354. ret['train'] = json.loads(ret['train'])
  355. return ret
  356. if 'pid' in data:
  357. from .project.task import get_default_params
  358. ret = get_default_params(data, SD.workspace, SD.machine_info)
  359. return ret
  360. if request.method == 'POST':
  361. from .project.task import set_task_params
  362. ret = set_task_params(data, SD.workspace)
  363. return ret
  364. @app.route('/project/task/metrics', methods=['GET'])
  365. def task_metrics():
  366. """
  367. methods=='GET':#获取日志数据
  368. Args:
  369. tid(str):任务id
  370. type(str):可以获取日志的类型,[train,eval,sensitivities,prune],包括训练,评估,敏感度与模型裁剪率关系图,裁剪的日志
  371. Return:
  372. status
  373. if type == 'train':
  374. train_log(dict): 训练日志
  375. elif type == 'eval':
  376. eval_metrics(dict): 评估结果
  377. elif type == 'sensitivities':
  378. sensitivities_loss_img(dict): 敏感度与模型裁剪率关系图
  379. elif type == 'prune':
  380. prune_log(dict):裁剪日志
  381. """
  382. data = request.get_json()
  383. if data is None:
  384. data = request.args
  385. if request.method == 'GET':
  386. if data['type'] == 'train':
  387. from .project.task import get_train_metrics
  388. ret = get_train_metrics(data, SD.workspace)
  389. return ret
  390. if data['type'] == 'eval':
  391. from .project.task import get_eval_metrics
  392. ret = get_eval_metrics(data, SD.workspace)
  393. return ret
  394. if data['type'] == 'eval_all':
  395. from .project.task import get_eval_all_metrics
  396. ret = get_eval_all_metrics(data, SD.workspace)
  397. return ret
  398. if data['type'] == 'sensitivities':
  399. from .project.task import get_sensitivities_loss_img
  400. ret = get_sensitivities_loss_img(data, SD.workspace)
  401. return ret
  402. if data['type'] == 'prune':
  403. from .project.task import get_prune_metrics
  404. ret = get_prune_metrics(data, SD.workspace)
  405. return ret
  406. @app.route('/project/task/train', methods=['POST', 'PUT'])
  407. def task_train():
  408. """
  409. methods=='POST':#异步,启动训练或者裁剪任务
  410. Args:
  411. tid(str):任务id
  412. eval_metric_loss(int,optional):可选,裁剪任务时可用,裁剪任务所需的评估loss
  413. Return:
  414. status
  415. methods=='PUT':#改变任务训练的状态,即终止训练或者恢复训练
  416. Args:
  417. tid(str):任务id
  418. act(str):[stop,resume]暂停或者恢复
  419. epoch(int):(resume下可以设置)恢复训练的起始轮数
  420. Return:
  421. status
  422. """
  423. data = request.get_json()
  424. if request.method == 'POST':
  425. from .project.task import start_train_task
  426. ret = start_train_task(data, SD.workspace, SD.monitored_processes)
  427. return ret
  428. if request.method == 'PUT':
  429. if data['act'] == 'resume':
  430. from .project.task import resume_train_task
  431. ret = resume_train_task(data, SD.workspace, SD.monitored_processes)
  432. return ret
  433. if data['act'] == 'stop':
  434. from .project.task import stop_train_task
  435. ret = stop_train_task(data, SD.workspace)
  436. return ret
  437. @app.route('/project/task/train/file', methods=['GET'])
  438. def log_file():
  439. data = request.get_json()
  440. if request.method == 'GET':
  441. path = data['path']
  442. if not os.path.exists(path):
  443. return {'status': -1}
  444. logs = open(path, encoding='utf-8').readlines()
  445. if len(logs) < 50:
  446. return {'status': 1, 'log': logs}
  447. else:
  448. logs = logs[-50:]
  449. return {'status': 1, 'log': logs}
  450. @app.route('/project/task/prune', methods=['GET', 'POST', 'PUT'])
  451. def task_prune():
  452. """
  453. methods=='GET':#获取裁剪任务的状态
  454. Args:
  455. tid(str):任务id
  456. Return:
  457. prune_status(int): 裁剪任务状态(PruneStatus)枚举变量的值
  458. status
  459. methods=='POST':#异步,创建一个裁剪分析,对于启动裁剪任务前需要先启动裁剪分析
  460. Args:
  461. tid(str):任务id
  462. Return:
  463. status
  464. methods=='PUT':#改变裁剪分析任务的状态
  465. Args:
  466. tid(str):任务id
  467. act(str):[stop],目前仅支持停止一个裁剪分析任务
  468. Return
  469. status
  470. """
  471. data = request.get_json()
  472. if data is None:
  473. data = request.args
  474. if request.method == 'GET':
  475. from .project.task import get_prune_status
  476. ret = get_prune_status(data, SD.workspace)
  477. return ret
  478. if request.method == 'POST':
  479. from .project.task import start_prune_analysis
  480. ret = start_prune_analysis(data, SD.workspace, SD.monitored_processes)
  481. return ret
  482. if request.method == 'PUT':
  483. if data['act'] == 'stop':
  484. from .project.task import stop_prune_analysis
  485. ret = stop_prune_analysis(data, SD.workspace)
  486. return ret
  487. @app.route('/project/task/evaluate', methods=['GET', 'POST'])
  488. def task_evaluate():
  489. '''
  490. methods=='GET':#获取模型评估的结果
  491. Args:
  492. tid(str):任务id
  493. Return:
  494. evaluate_status(int): 任务状态(TaskStatus)枚举变量的值
  495. message(str):描述评估任务的信息
  496. result(dict):如果评估成功,返回评估结果的dict,否则为None
  497. status
  498. methods=='POST':#异步,创建一个评估任务
  499. Args:
  500. tid(str):任务id
  501. epoch(int,optional):需要评估的epoch,如果为None则会评估训练时指标最好的epoch
  502. topk(int,optional):分类任务topk指标,如果为None默认输入为5
  503. score_thresh(float):检测任务类别的score threshhold值,如果为None默认输入为0.5
  504. overlap_thresh(float):实例分割任务IOU threshhold值,如果为None默认输入为0.3
  505. Return:
  506. status
  507. '''
  508. data = request.get_json()
  509. if data is None:
  510. data = request.args
  511. if request.method == 'GET':
  512. from .project.task import get_evaluate_result
  513. ret = get_evaluate_result(data, SD.workspace)
  514. if ret['evaluate_status'] == TaskStatus.XEVALUATED and ret[
  515. 'result'] is not None:
  516. if 'Confusion_Matrix' in ret['result']:
  517. ret['result']['Confusion_Matrix'] = ret['result'][
  518. 'Confusion_Matrix'].tolist()
  519. ret['result'] = CustomEncoder().encode(ret['result'])
  520. ret['result'] = json.loads(ret['result'])
  521. ret['evaluate_status'] = ret['evaluate_status'].value
  522. return ret
  523. if request.method == 'POST':
  524. from .project.task import evaluate_model
  525. ret = evaluate_model(data, SD.workspace, SD.monitored_processes)
  526. return ret
  527. @app.route('/project/task/evaluate/file', methods=['GET'])
  528. def task_evaluate_file():
  529. data = request.get_json()
  530. if request.method == 'GET':
  531. if 'path' in data:
  532. ret = data['path']
  533. return send_file(ret)
  534. else:
  535. from .project.task import get_evaluate_result
  536. from .project.task import import_evaluate_excel
  537. ret = get_evaluate_result(data, SD.workspace)
  538. if ret['evaluate_status'] == TaskStatus.XEVALUATED and ret[
  539. 'result'] is not None:
  540. result = ret['result']
  541. excel_ret = dict()
  542. excel_ret = import_evaluate_excel(data, result, SD.workspace)
  543. return excel_ret
  544. else:
  545. excel_ret = dict()
  546. excel_ret['path'] = None
  547. excel_ret['status'] = -1
  548. excel_ret['message'] = "评估尚未完成或评估失败"
  549. return excel_ret
  550. @app.route('/project/task/predict', methods=['GET', 'POST', 'PUT'])
  551. def task_predict():
  552. '''
  553. methods=='GET':#获取预测状态
  554. Args:
  555. tid(str):任务id
  556. Return:
  557. predict_status(int): 预测任务状态(PredictStatus)枚举变量的值
  558. message(str): 预测信息
  559. status
  560. methods=='POST':#创建预测任务,目前仅支持单张图片的预测
  561. Args:
  562. tid(str):任务id
  563. image_data(str):base64编码的image数据
  564. score_thresh(float,optional):可选,检测任务时有效,检测类别的score threashold值默认是0.5
  565. epoch(int,float,optional):可选,选择需要做预测的ephoch,默认为评估指标最好的那一个epoch
  566. Return:
  567. path(str):服务器上保存预测结果图片的路径
  568. status
  569. '''
  570. data = request.get_json()
  571. if data is None:
  572. data = request.args
  573. if request.method == 'GET':
  574. from .project.task import get_predict_status
  575. ret = get_predict_status(data, SD.workspace)
  576. return ret
  577. if request.method == 'POST':
  578. from .project.task import predict_test_pics
  579. ret = predict_test_pics(data, SD.workspace, SD.monitored_processes)
  580. if 'img_list' in data:
  581. del ret['path']
  582. return ret
  583. return ret
  584. if request.method == 'PUT':
  585. from .project.task import stop_predict_task
  586. ret = stop_predict_task(data, SD.workspace)
  587. return ret
  588. @app.route('/project/task/export', methods=['GET', 'POST', 'PUT'])
  589. def task_export():
  590. '''
  591. methods=='GET':#获取导出模型的状态
  592. Args:
  593. tid(str):任务id
  594. quant(str,optional)可选,[log,result],导出量模型导出状态,若值为log则返回量化的日志;若值为result则返回量化的结果
  595. Return:
  596. status
  597. if quant == 'log':
  598. quant_log(dict):量化日志
  599. if quant == 'result'
  600. quant_result(dict):量化结果
  601. if quant not in Args:
  602. export_status(int):模型导出状态(PredictStatus)枚举变量的值
  603. message(str):模型导出提示信息
  604. methods=='POST':#导出inference模型或者导出lite模型
  605. Args:
  606. tid(str):任务id
  607. type(str):保存模型的类别[infer,lite],支持inference模型导出和lite的模型导出
  608. save_dir(str):保存模型的路径
  609. epoch(str,optional)可选,指定导出的epoch数默认为评估效果最好的epoch
  610. quant(bool,optional)可选,type为infer有效,是否导出量化后的模型,默认为False
  611. model_path(str,optional)可选,type为lite时有效,inference模型的地址
  612. Return:
  613. status
  614. if type == 'infer':
  615. save_dir:模型保存路径
  616. if type == 'lite':
  617. message:模型保存信息
  618. methods=='PUT':#停止导出模型
  619. Args:
  620. tid(str):任务id
  621. Return:
  622. export_status(int):模型导出状态(PredictStatus)枚举变量的值
  623. message(str):停止模型导出提示信息
  624. status
  625. '''
  626. data = request.get_json()
  627. if data is None:
  628. data = request.args
  629. if request.method == 'GET':
  630. if 'quant' in data:
  631. if data['quant'] == 'log':
  632. from .project.task import get_quant_progress
  633. ret = get_quant_progress(data, SD.workspace)
  634. return ret
  635. if data['quant'] == 'result':
  636. from .project.task import get_quant_result
  637. ret = get_quant_result(data, SD.workspace)
  638. return ret
  639. from .project.task import get_export_status
  640. ret = get_export_status(data, SD.workspace)
  641. ret['export_status'] = ret['export_status'].value
  642. return ret
  643. if request.method == 'POST':
  644. if data['type'] == 'infer':
  645. from .project.task import export_infer_model
  646. ret = export_infer_model(data, SD.workspace,
  647. SD.monitored_processes)
  648. return ret
  649. if data['type'] == 'lite':
  650. from .project.task import export_lite_model
  651. ret = export_lite_model(data, SD.workspace)
  652. return ret
  653. if request.method == 'PUT':
  654. from .project.task import stop_export_task
  655. stop_export_task(data, SD.workspace)
  656. return ret
  657. @app.route('/project/task/vdl', methods=['GET'])
  658. def task_vdl():
  659. '''
  660. methods=='GET':#打开某个任务的可视化分析工具(VisualDL)
  661. Args:
  662. tid(str):任务id
  663. Return:
  664. url(str):vdl地址
  665. status
  666. '''
  667. data = request.get_json()
  668. if data is None:
  669. data = request.args
  670. if request.method == 'GET':
  671. from .project.task import open_vdl
  672. ret = open_vdl(data, SD.workspace, SD.current_port,
  673. SD.monitored_processes, SD.running_boards)
  674. return ret
  675. @app.route('/system', methods=['GET', 'DELETE'])
  676. def system():
  677. '''
  678. methods=='GET':#获取系统GPU、CPU信息
  679. Args:
  680. type(str):[machine_info,gpu_memory_size]选择需要获取的系统信息
  681. Return:
  682. status
  683. if type=='machine_info'
  684. info(dict):服务端信息
  685. if type=='gpu_memory_size'
  686. gpu_mem_infos(list):GPU内存信息
  687. '''
  688. data = request.get_json()
  689. if data is None:
  690. data = request.args
  691. if request.method == 'GET':
  692. if data['type'] == 'machine_info':
  693. '''if 'path' not in data:
  694. data['path'] = None
  695. from .system import get_machine_info
  696. ret = get_machine_info(data, SD.machine_info)'''
  697. from .system import get_system_info
  698. ret = get_system_info(SD.machine_info)
  699. return ret
  700. if data['type'] == 'gpu_memory_size':
  701. #from .system import get_gpu_memory_size
  702. from .system import get_gpu_memory_info
  703. ret = get_gpu_memory_info(SD.machine_info)
  704. return ret
  705. if request.method == 'DELETE':
  706. from .system import exit_system
  707. ret = exit_system(SD.monitored_processes)
  708. return ret
  709. @app.route('/demo', methods=['GET', 'POST', 'PUT'])
  710. def demo():
  711. '''
  712. methods=='GET':#获取demo下载进度
  713. Args:
  714. prj_type(int):项目类型ProjectType枚举变量的int值
  715. Return:
  716. status
  717. attr(dict):demo下载信息
  718. methods=='POST':#下载或创建demo工程
  719. Args:
  720. type(str):{download,load}下载或者创建样例
  721. prj_type(int):项目类型ProjectType枚举变量的int值
  722. Return:
  723. status
  724. if type=='load':
  725. did:数据集id
  726. pid:项目id
  727. methods=='PUT':#停止下载或创建demo工程
  728. Args:
  729. prj_type(int):项目类型ProjectType枚举变量的int值
  730. Return:
  731. status
  732. '''
  733. data = request.get_json()
  734. if data is None:
  735. data = request.args
  736. if request.method == 'GET':
  737. from .demo import get_download_demo_progress
  738. ret = get_download_demo_progress(data, SD.workspace)
  739. return ret
  740. if request.method == 'POST':
  741. if data['type'] == 'download':
  742. from .demo import download_demo_dataset
  743. ret = download_demo_dataset(data, SD.workspace,
  744. SD.load_demo_proc_dict)
  745. return ret
  746. if data['type'] == 'load':
  747. from .demo import load_demo_project
  748. ret = load_demo_project(data, SD.workspace, SD.monitored_processes,
  749. SD.load_demo_proj_data_dict,
  750. SD.load_demo_proc_dict)
  751. return ret
  752. if request.method == 'PUT':
  753. from .demo import stop_import_demo
  754. ret = stop_import_demo(data, SD.workspace, SD.load_demo_proc_dict,
  755. SD.load_demo_proj_data_dict)
  756. return ret
  757. @app.route('/model', methods=['GET', 'POST', 'DELETE'])
  758. def model():
  759. '''
  760. methods=='GET':#获取一个或者所有模型的信息
  761. Args:
  762. mid(str,optional)可选,若存在则返回某个模型的信息
  763. type(str,optional)可选,[pretrained,exported].若存在则返回对应类型下所有的模型信息
  764. Return:
  765. status
  766. if mid in Args:
  767. dataset_attr(dict):数据集属性
  768. task_params(dict):模型训练参数
  769. eval_result(dict):模型评估结果
  770. if type in Args and type == 'pretrained':
  771. pretrained_models(list):所有预训练模型信息
  772. if type in Args and type == 'exported':
  773. exported_models(list):所有inference模型的信息
  774. methods=='POST':#创建一个模型
  775. Args:
  776. pid(str):项目id
  777. tid(str):任务id
  778. name(str):模型名字
  779. type(str):创建模型的类型,[pretrained,exported],pretrained代表创建预训练模型、exported代表创建inference或者lite模型
  780. source_path(str):仅type为pretrained时有效,训练好的模型的路径
  781. path(str):仅type为exported时有效,inference或者lite模型的路径
  782. exported_type(int):0为inference模型,1为lite模型
  783. eval_results(dict,optional):可选,仅type为pretrained时有效,模型评估的指标
  784. Return:
  785. status
  786. if type == 'pretrained':
  787. pmid(str):预训练模型id
  788. if type == 'exported':
  789. emid(str):inference模型id
  790. methods=='DELETE':删除一个模型
  791. Args:
  792. type(str):删除模型的类型,[pretrained,exported],pretrained代表创建预训练模型、exported代表创建inference或者lite模型
  793. if type='pretrained':
  794. pmid:预训练模型id
  795. if type='exported':
  796. emid:inference或者lite模型id
  797. Return:
  798. status
  799. '''
  800. data = request.get_json()
  801. if data is None:
  802. data = request.args
  803. if request.method == 'GET':
  804. if 'type' in data:
  805. if data['type'] == 'pretrained':
  806. from .model import list_pretrained_models
  807. ret = list_pretrained_models(SD.workspace)
  808. return ret
  809. if data['type'] == 'exported':
  810. from .model import list_exported_models
  811. ret = list_exported_models(SD.workspace)
  812. return ret
  813. from .model import get_model_details
  814. ret = get_model_details(data, SD.workspace)
  815. ret['eval_result']['Confusion_Matrix'] = ret['eval_result'][
  816. 'Confusion_Matrix'].tolist()
  817. ret['eval_result'] = CustomEncoder().encode(ret['eval_result'])
  818. ret['task_params'] = CustomEncoder().encode(ret['task_params'])
  819. return ret
  820. if request.method == 'POST':
  821. if data['type'] == 'pretrained':
  822. if 'eval_results' in data:
  823. data['eval_results']['Confusion_Matrix'] = np.array(data[
  824. 'eval_results']['Confusion_Matrix'])
  825. from .model import create_pretrained_model
  826. ret = create_pretrained_model(data, SD.workspace,
  827. SD.monitored_processes)
  828. return ret
  829. if data['type'] == 'exported':
  830. from .model import create_exported_model
  831. ret = create_exported_model(data, SD.workspace)
  832. return ret
  833. if request.method == 'DELETE':
  834. if data['type'] == 'pretrained':
  835. from .model import delete_pretrained_model
  836. ret = delete_pretrained_model(data, SD.workspace)
  837. return ret
  838. if data['type'] == 'exported':
  839. from .model import delete_exported_model
  840. ret = delete_exported_model(data, SD.workspace)
  841. return ret
  842. @app.route('/model/file', methods=['GET'])
  843. def model_file():
  844. data = request.get_json()
  845. if request.method == 'GET':
  846. ret = data['path']
  847. return send_file(ret)
  848. def run(port, workspace_dir):
  849. if workspace_dir is None:
  850. user_home = os.path.expanduser('~')
  851. dirname = osp.join(user_home, "paddlex_workspace")
  852. else:
  853. dirname = workspace_dir
  854. if not osp.exists(dirname):
  855. os.makedirs(dirname)
  856. else:
  857. if not osp.isdir(dirname):
  858. os.remove(dirname)
  859. os.makedirs(dirname)
  860. logger = get_logger(osp.join(dirname, "mcessages.log"))
  861. init(dirname, logger)
  862. try:
  863. app.run(host='0.0.0.0', port=port, threaded=True)
  864. except:
  865. print("请确保端口号:{}未被防火墙限制".format(port))